From abdb1af77f7081f78a8631611d18dcef7bdf1590 Mon Sep 17 00:00:00 2001 From: Drew Short Date: Thu, 6 Feb 2014 16:51:22 -0500 Subject: [PATCH] Moved logging primarily over to the typesafe:config system which uses JSON style nested configs. Also fixed some logging issues. --- pom.xml | 9 +- src/includes/logback.xml | 2 +- .../java/com/sothr/imagetools/AppCLI.java | 1 + .../java/com/sothr/imagetools/AppConfig.java | 6 +- src/main/resources/application.conf | 75 +++++---- .../imagetools/util/PropertiesEnum.scala | 38 +++-- .../imagetools/util/PropertiesService.scala | 143 +++++++++--------- src/test/resources/application.conf | 116 +++++++------- todo | 15 +- 9 files changed, 195 insertions(+), 210 deletions(-) diff --git a/pom.xml b/pom.xml index b87df82..6694516 100644 --- a/pom.xml +++ b/pom.xml @@ -44,6 +44,8 @@ 1.2 1.9 2.4.0 + 1.2.0 + [0.4, 0.5) @@ -92,7 +94,12 @@ net.coobird thumbnailator - [0.4, 0.5) + ${lib.thumbnailator.version} + + + com.typesafe + config + ${lib.typesafe-config.version} net.sourceforge.jtransforms diff --git a/src/includes/logback.xml b/src/includes/logback.xml index 4a298fd..713ce5d 100644 --- a/src/includes/logback.xml +++ b/src/includes/logback.xml @@ -4,7 +4,7 @@ false - [%-5level [%c{16}] - %message%n + [%date{HH:mm:ss}] %-5level [%c{16}] - %message%n INFO diff --git a/src/main/java/com/sothr/imagetools/AppCLI.java b/src/main/java/com/sothr/imagetools/AppCLI.java index 0a20ad9..6014b3e 100644 --- a/src/main/java/com/sothr/imagetools/AppCLI.java +++ b/src/main/java/com/sothr/imagetools/AppCLI.java @@ -26,6 +26,7 @@ class AppCLI { CommandLineParser parser = new BasicParser(); CommandLine cmd = parser.parse(options, args); process(cmd); + AppConfig.saveProperties(); System.exit(0); } catch (Exception ex) { logger.error("Unhandled exception in AppCLI",ex); diff --git a/src/main/java/com/sothr/imagetools/AppConfig.java b/src/main/java/com/sothr/imagetools/AppConfig.java index 366250d..fd5f6a7 100644 --- a/src/main/java/com/sothr/imagetools/AppConfig.java +++ b/src/main/java/com/sothr/imagetools/AppConfig.java @@ -26,8 +26,8 @@ public class AppConfig { private static Boolean configuredLogging = false; //Properties defaults - private static final String DEFAULTPROPERTIESFILE = "default.properties"; - private static final String USERPROPERTIESFILE = "./config.xml"; + private static final String DEFAULTPROPERTIESFILE = "application.conf"; + private static final String USERPROPERTIESFILE = "user.conf"; private static Boolean loadedProperties = false; //Cache defaults @@ -110,7 +110,7 @@ public class AppConfig { } public static void saveProperties() { - PropertiesService.saveXMLProperties(USERPROPERTIESFILE); + PropertiesService.saveConf(USERPROPERTIESFILE); logger.debug("Saved properties"); } diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index d2fcb02..9ae72b5 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -1,6 +1,5 @@ //Default Properties File //Image Tools version: ${project.version} -version = "${build-version}" akka { loggers = ["akka.event.slf4j.Slf4jLogger"] @@ -9,12 +8,8 @@ akka { //Default App Settings app { + version.current = "${build-version}" timed = false - log { - debug = false - info = false - error = true - } engine { //Concurrency Settings concurrent { @@ -22,41 +17,39 @@ app { processing.limit = 15 } } -} - -#Default Image Settings -image { - //images must be 90% similar - differenceThreshold = 0.90 - //control generation of hashes for new images. - hash { - precision=64 - } - ahash { - use = true - weight = 0.70 - precision = 8 - tolerence = 8 - } - dhash { - use = true - weight = 0.85 - precision = 8 - tolerence = 8 + #Default Image Settings + image { + //images must be 90% similar + differenceThreshold = 0.90 + //control generation of hashes for new images. + hash { + precision=64 + } + ahash { + use = true + weight = 0.70 + precision = 8 + tolerence = 8 + } + dhash { + use = true + weight = 0.85 + precision = 8 + tolerence = 8 + } + phash { + //set to false if hashing images is taking too long + use = true + weight = 1.0 + precision = 32 + tolerence = 8 + } } - phash { - //set to false if hashing images is taking too long - use = true - weight = 1.0 - precision = 32 - tolerence = 8 + //Default Thumbnail Settings + thumbnail { + //Directory where to store thumbnails + directory = ".cache/thumbnails/" + //Size of the thumbnail to generate and store + size = 128 } -} - -//Default Thumbnail Settings -thumbnail { - //Directory where to store thumbnails - directory = ".cache/thumbnails/" - //Size of the thumbnail to generate and store - size = 128 } \ No newline at end of file diff --git a/src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala b/src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala index 0ccb433..b449ecf 100644 --- a/src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala +++ b/src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala @@ -2,31 +2,29 @@ package com.sothr.imagetools.util object PropertiesEnum extends Enumeration { type PropertiesEnum = Value - val Version = Value("version") + val Version = Value("app.version.current") + val PreviousVersion = Value("app.version.previous") //default app settings - val LogDebug = Value("app.log.debug") - val LogInfo = Value("app.log.info") - val LogError = Value("app.log.error") val Timed = Value("app.timed") //default engine concurrency settings val ConcurrentSimiliartyLimit = Value("app.engine.concurrent.similarity.limit") val ConcurrentProcessingLimit = Value("app.engine.concurrent.processing.limit") //default image settings - val ImageDifferenceThreshold = Value("image.differenceThreshold") - val HashPrecision = Value("image.hash.precision") - val UseAhash = Value("image.ahash.use") - val AhashWeight = Value("image.ahash.weight") - val AhashPrecision = Value("image.ahash.precision") - val AhashTolerance = Value("image.ahash.tolerence") - val UseDhash = Value("image.dhash.use") - val DhashWeight = Value("image.dhash.weight") - val DhashPrecision = Value("image.dhash.precision") - val DhashTolerance = Value("image.dhash.tolerence") - val UsePhash = Value("image.phash.use") - val PhashWeight = Value("image.phash.weight") - val PhashPrecision = Value("image.phash.precision") - val PhashTolerance = Value("image.phash.tolerence") + val ImageDifferenceThreshold = Value("app.image.differenceThreshold") + val HashPrecision = Value("app.image.hash.precision") + val UseAhash = Value("app.image.ahash.use") + val AhashWeight = Value("app.image.ahash.weight") + val AhashPrecision = Value("app.image.ahash.precision") + val AhashTolerance = Value("app.image.ahash.tolerence") + val UseDhash = Value("app.image.dhash.use") + val DhashWeight = Value("app.image.dhash.weight") + val DhashPrecision = Value("app.image.dhash.precision") + val DhashTolerance = Value("app.image.dhash.tolerence") + val UsePhash = Value("app.image.phash.use") + val PhashWeight = Value("app.image.phash.weight") + val PhashPrecision = Value("app.image.phash.precision") + val PhashTolerance = Value("app.image.phash.tolerence") //Default Thumbnail Settings - val ThumbnailDirectory = Value("thumbnail.directory") - val ThumbnailSize = Value("thumbnail.size") + val ThumbnailDirectory = Value("app.thumbnail.directory") + val ThumbnailSize = Value("app.thumbnail.size") } \ No newline at end of file diff --git a/src/main/scala/com/sothr/imagetools/util/PropertiesService.scala b/src/main/scala/com/sothr/imagetools/util/PropertiesService.scala index 103f553..658724b 100644 --- a/src/main/scala/com/sothr/imagetools/util/PropertiesService.scala +++ b/src/main/scala/com/sothr/imagetools/util/PropertiesService.scala @@ -1,8 +1,9 @@ package com.sothr.imagetools.util -import java.util.Properties +import com.typesafe.config.{Config, ConfigFactory} import grizzled.slf4j.Logging -import java.io.{FileInputStream, FileOutputStream, OutputStream} +import java.io.{File, PrintStream, FileOutputStream} +import java.util.Properties import scala.collection.JavaConversions._ /* @@ -10,107 +11,109 @@ import scala.collection.JavaConversions._ */ object PropertiesService extends Logging { - private val properties:Properties = new Properties() + private var defaultConf:Config = null + private var userConf:Config = null + private var newUserConf:Properties = new Properties() private var version:Version = null def getVersion:Version = this.version - private val propertiesToClean:Array[String] = Array("version") + + //specific highly used properties + var TimingEnabled:Boolean = false + + //ahash + var aHashPrecision = 0 + var aHashTolerance = 0 + var aHashWeight = 0.0f + var useAhash = false + //dhash + var dHashPrecision = 0 + var dHashTolerance = 0 + var dHashWeight = 0.0f + var useDhash = false + //phash + var pHashPrecision = 0 + var pHashTolerance = 0 + var pHashWeight = 0.0f + var usePhash = false /* * Load the properties file from the specified location */ def loadProperties(defaultLocation:String, userLocation:String = null) = { info(s"Attempting to load properties from: $defaultLocation") - val defaultInputStream = ResourceLoader.get.getResourceStream(defaultLocation) - properties.load(defaultInputStream) + defaultConf = ConfigFactory.load(defaultLocation); if (userLocation != null) { - val userInputStream = new FileInputStream(userLocation) - val userProperties = new Properties(); - userProperties.loadFromXML(userInputStream) - - for (propertyName:String <- userProperties.stringPropertyNames()) { - properties.setProperty(propertyName, userProperties.getProperty(propertyName)); - } + userConf = ConfigFactory.parseFile(new File(userLocation)); } else { + userConf = ConfigFactory.empty info("No user properties file exists to load from") } - version = new Version(properties.getProperty("version")); + version = new Version(get(PropertiesEnum.Version.toString)); info(s"Detected Version: $version") //load special properties - DebugLogEnabled = get(PropertiesEnum.LogDebug.toString).toBoolean - InfoLogEnabled = get(PropertiesEnum.LogInfo.toString).toBoolean - ErrorLogEnabled = get(PropertiesEnum.LogError.toString).toBoolean TimingEnabled = get(PropertiesEnum.Timed.toString).toBoolean //ahash - aHashPrecision = PropertiesService.get(PropertiesEnum.AhashPrecision.toString).toInt - aHashTolerance = PropertiesService.get(PropertiesEnum.AhashTolerance.toString).toInt - aHashWeight = PropertiesService.get(PropertiesEnum.AhashWeight.toString).toFloat - useAhash = PropertiesService.get(PropertiesEnum.UseAhash.toString).toBoolean + aHashPrecision = get(PropertiesEnum.AhashPrecision.toString).toInt + aHashTolerance = get(PropertiesEnum.AhashTolerance.toString).toInt + aHashWeight = get(PropertiesEnum.AhashWeight.toString).toFloat + useAhash = get(PropertiesEnum.UseAhash.toString).toBoolean //dhash - dHashPrecision = PropertiesService.get(PropertiesEnum.DhashPrecision.toString).toInt - dHashTolerance = PropertiesService.get(PropertiesEnum.DhashTolerance.toString).toInt - dHashWeight = PropertiesService.get(PropertiesEnum.DhashWeight.toString).toFloat - useDhash = PropertiesService.get(PropertiesEnum.UseDhash.toString).toBoolean + dHashPrecision = get(PropertiesEnum.DhashPrecision.toString).toInt + dHashTolerance = get(PropertiesEnum.DhashTolerance.toString).toInt + dHashWeight = get(PropertiesEnum.DhashWeight.toString).toFloat + useDhash = get(PropertiesEnum.UseDhash.toString).toBoolean //phash - pHashPrecision = PropertiesService.get(PropertiesEnum.PhashPrecision.toString).toInt - pHashTolerance = PropertiesService.get(PropertiesEnum.PhashTolerance.toString).toInt - pHashWeight = PropertiesService.get(PropertiesEnum.PhashWeight.toString).toFloat - usePhash = PropertiesService.get(PropertiesEnum.UsePhash.toString).toBoolean + pHashPrecision = get(PropertiesEnum.PhashPrecision.toString).toInt + pHashTolerance = get(PropertiesEnum.PhashTolerance.toString).toInt + pHashWeight = get(PropertiesEnum.PhashWeight.toString).toFloat + usePhash = get(PropertiesEnum.UsePhash.toString).toBoolean info("Loaded Special Properties") } - /** - * Gets a properties object that is cleaned of things that are expected to change. i.e. version - */ - private def getCleanProperties():Properties = { - val cleanProperties:Properties = properties.clone().asInstanceOf[Properties] - //Remove properties to be cleaned - for (key <- propertiesToClean) { - cleanProperties.remove(key) - } - return cleanProperties + private def cleanAndPrepareNewUserProperties():Properties = { + //insert special keys here + newUserConf.setProperty(PropertiesEnum.PreviousVersion.toString, version.parsableToString()) + //remove special keys here + newUserConf.remove(PropertiesEnum.Version.toString) + newUserConf } - def saveXMLProperties(location:String) = { + private def getCleanedMergedUserConf():Config = { + + ConfigFactory.parseProperties(cleanAndPrepareNewUserProperties()) withFallback(userConf) + } + + def saveConf(location:String) = { info(s"Saving user properties to $location") - val out:OutputStream = new FileOutputStream(location, false) - val cleanProperties = getCleanProperties - //insert special keys here - cleanProperties.setProperty("version.previous", version.parsableToString()) - cleanProperties.storeToXML(out, "User Properties") + val out:PrintStream = new PrintStream(new FileOutputStream(location, false)) + val userConfToSave = getCleanedMergedUserConf + //print to the output stream + out.print(userConfToSave.root.render) out.flush() out.close() } - def get(key:String):String = { - return properties.getProperty(key) + def get(key:String, defaultValue:String=null):String = { + var result:String = defaultValue + //check the latest properties + if (newUserConf.containsKey(key)) { + result = newUserConf.getProperty(key) + } + //check the loaded user properties + else if (userConf.hasPath(key)) { + result = userConf.getString(key) + } + //check the default properties + else if (defaultConf.hasPath(key)) { + result = defaultConf.getString(key) + } + return result } def set(key:String, value:String) = { - properties.setProperty(key, value) + newUserConf.setProperty(key, value) } - - //specific highly used properties - var DebugLogEnabled:Boolean = false - var InfoLogEnabled:Boolean = false - var ErrorLogEnabled:Boolean = false - var TimingEnabled:Boolean = false - - //ahash - var aHashPrecision = 0 - var aHashTolerance = 0 - var aHashWeight = 0.0f - var useAhash = false - //dhash - var dHashPrecision = 0 - var dHashTolerance = 0 - var dHashWeight = 0.0f - var useDhash = false - //phash - var pHashPrecision = 0 - var pHashTolerance = 0 - var pHashWeight = 0.0f - var usePhash = false } \ No newline at end of file diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf index 6fab465..5d0a916 100644 --- a/src/test/resources/application.conf +++ b/src/test/resources/application.conf @@ -1,63 +1,55 @@ -//Default Properties File -//Image Tools version: ${project.version} -version = "${build-version}" - - -akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] - loglevel = "DEBUG" -} - -//Default App Settings -app { - timed = false - log { - debug = true - info = true - error = true - } - engine { - //Concurrency Settings - concurrent { - similarity.limit = 15 - processing.limit = 15 - } - } -} - -#Default Image Settings -image { - //images must be 90% similar - differenceThreshold = 0.90 - //control generation of hashes for new images. - hash { - precision=64 - } - ahash { - use = true - weight = 0.70 - precision = 8 - tolerence = 8 - } - dhash { - use = true - weight = 0.85 - precision = 8 - tolerence = 8 - } - phash { - //set to false if hashing images is taking too long - use = true - weight = 1.0 - precision = 32 - tolerence = 8 - } -} - -//Default Thumbnail Settings -thumbnail { - //Directory where to store thumbnails - directory = ".cache/thumbnails/" - //Size of the thumbnail to generate and store - size = 128 +//Default Properties File +//Image Tools version: ${project.version} + +akka { + loggers = ["akka.event.slf4j.Slf4jLogger"] + loglevel = "INFO" +} + +//Default App Settings +app { + version.current = "${build-version}" + timed = false + engine { + //Concurrency Settings + concurrent { + similarity.limit = 2 + processing.limit = 2 + } + } + #Default Image Settings + image { + //images must be 90% similar + differenceThreshold = 0.90 + //control generation of hashes for new images. + hash { + precision=64 + } + ahash { + use = true + weight = 0.70 + precision = 8 + tolerence = 8 + } + dhash { + use = true + weight = 0.85 + precision = 8 + tolerence = 8 + } + phash { + //set to false if hashing images is taking too long + use = true + weight = 1.0 + precision = 32 + tolerence = 8 + } + } + //Default Thumbnail Settings + thumbnail { + //Directory where to store thumbnails + directory = ".cache/thumbnails/" + //Size of the thumbnail to generate and store + size = 128 + } } \ No newline at end of file diff --git a/todo b/todo index c5f252e..a160966 100644 --- a/todo +++ b/todo @@ -1,18 +1,9 @@ -Move over to Logback for the backend - -Convert the configuration file - -Convert the AppConfig to configure basic logging -Profile and improve performance for the ConcurrentEngine when looking for similarities. - -SynchronousEngine is about 2.5x faster currently for finding similarities - -ConcurrentEngine actors for finding similarities are not CPU constrained and need to be -Convert app configuration format from a java properties to a typesafehub:config - -Rewrite properties in new format (JSON) - -Rewrite PropertiesLoader to use the new format Add functionality to both engines - -Recursive directory scanning - -Determine how to handle similarites there... All images or just per folder -Move files to new locations -Rename new files based on their MD5 Add functionality to ImageService -Cache thumbnails -Generate thumbnails - -Move caching of images into the ImageService \ No newline at end of file +Improve functionality of the PropertiesService + -Getters and Setters for sepcific values with proper cascading and type awareness + -Nicer debugging? \ No newline at end of file