From 0fcad44e4e7a417a1e5f4562f86d185f78e82ccd Mon Sep 17 00:00:00 2001 From: Drew Short Date: Fri, 7 Feb 2014 15:59:00 -0500 Subject: [PATCH] Started work on database integration. Cleaned up caching of thumbnails and improved performance of the thumb path method. --- pom.xml | 55 ++++++++---- src/main/resources/application.conf | 87 ++++++++++--------- src/main/resources/hibernate.cfg.xml | 34 ++++++++ .../sothr/imagetools/image/ImageService.scala | 23 +---- src/test/resources/application.conf | 87 ++++++++++--------- todo | 9 +- 6 files changed, 174 insertions(+), 121 deletions(-) create mode 100644 src/main/resources/hibernate.cfg.xml diff --git a/pom.xml b/pom.xml index 6694516..4749f7f 100644 --- a/pom.xml +++ b/pom.xml @@ -26,9 +26,6 @@ - 0 - 1 - DEV UTF-8 1.7 3.8.1 @@ -46,6 +43,8 @@ 2.4.0 1.2.0 [0.4, 0.5) + 1.3.175 + 4.3.0.Final @@ -62,19 +61,19 @@ test - ch.qos.logback - logback-core - ${lib.logback.version} + ch.qos.logback + logback-core + ${lib.logback.version} - ch.qos.logback - logback-classic - ${lib.logback.version} + ch.qos.logback + logback-classic + ${lib.logback.version} - ch.qos.logback - logback-access - ${lib.logback.version} + ch.qos.logback + logback-access + ${lib.logback.version} org.slf4j @@ -107,9 +106,9 @@ ${lib.jtransforms.version} - commons-cli - commons-cli - ${lib.commons-cli.version} + commons-cli + commons-cli + ${lib.commons-cli.version} commons-codec @@ -132,9 +131,24 @@ ${lib.akka.version} - com.typesafe.akka - akka-slf4j_2.10 - ${lib.akka.version} + com.typesafe.akka + akka-slf4j_2.10 + ${lib.akka.version} + + + com.h2database + h2 + ${lib.h2database.version} + + + org.hibernate + hibernate-core + ${lib.hibernate.version} + + + org.hibernate + hibernate-ehcache + ${lib.hibernate.version} @@ -336,7 +350,10 @@ process-resources - + + + + diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 9ae72b5..22abb13 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -8,48 +8,53 @@ akka { //Default App Settings app { - version.current = "${build-version}" - timed = false - engine { - //Concurrency Settings - concurrent { - similarity.limit = 15 - processing.limit = 15 - } + version.current = "${build-version}" + timed = false + engine { + //Concurrency Settings + concurrent { + similarity.limit = 15 + processing.limit = 15 } - #Default Image Settings - image { - //images must be 90% similar - differenceThreshold = 0.90 - //control generation of hashes for new images. - hash { - precision=64 - } - ahash { - use = true - weight = 0.70 - precision = 8 - tolerence = 8 - } - dhash { - use = true - weight = 0.85 - precision = 8 - tolerence = 8 - } - phash { - //set to false if hashing images is taking too long - use = true - weight = 1.0 - precision = 32 - tolerence = 8 - } + } + #Default Image Settings + image { + //images must be 90% similar + differenceThreshold = 0.90 + //control generation of hashes for new images. + hash { + precision=64 } - //Default Thumbnail Settings - thumbnail { - //Directory where to store thumbnails - directory = ".cache/thumbnails/" - //Size of the thumbnail to generate and store - size = 128 + ahash { + use = true + weight = 0.70 + precision = 8 + tolerence = 8 } + dhash { + use = true + weight = 0.85 + precision = 8 + tolerence = 8 + } + phash { + //set to false if hashing images is taking too long + use = true + weight = 1.0 + precision = 32 + tolerence = 8 + } + } + //Default Thumbnail Settings + thumbnail { + //Directory where to store thumbnails + directory = ".cache/thumbnails/" + //Size of the thumbnail to generate and store + size = 128 + } + //Default Database Settings + database { + location = ".cache/database.db" + inMemory = false + } } \ No newline at end of file diff --git a/src/main/resources/hibernate.cfg.xml b/src/main/resources/hibernate.cfg.xml new file mode 100644 index 0000000..6682cee --- /dev/null +++ b/src/main/resources/hibernate.cfg.xml @@ -0,0 +1,34 @@ + + + + + + + + org.h2.Driver + jdbc:h2:.cache/database.db + org.hibernate.dialect.H2Dialect + false + + org.hibernate.transaction.JTATransactionFactory + + java:comp/UserTransaction + + + thread + + + org.hibernate.cache.NoCacheProvider + + + + + + + + \ No newline at end of file diff --git a/src/main/scala/com/sothr/imagetools/image/ImageService.scala b/src/main/scala/com/sothr/imagetools/image/ImageService.scala index 1c1421d..4524269 100644 --- a/src/main/scala/com/sothr/imagetools/image/ImageService.scala +++ b/src/main/scala/com/sothr/imagetools/image/ImageService.scala @@ -14,7 +14,6 @@ import com.sothr.imagetools.util.{PropertiesEnum, PropertiesService} object ImageService extends Logging { val imageCache = AppConfig.cacheManager.getCache("images") - val thumbnailCache = AppConfig.cacheManager.getCache("thumbnails") private def lookupImage(file:File):Image = { var image:Image = null @@ -45,7 +44,7 @@ object ImageService extends Logging { val bufferedImage = ImageIO.read(file) val hashes = HashService.getImageHashes(bufferedImage, file.getAbsolutePath) var thumbnailPath = lookupThumbnailPath(hashes.md5) - if (thumbnailPath == null) thumbnailPath = saveThumbnail(hashes.md5, getThumbnail(bufferedImage, hashes.md5)) + if (thumbnailPath == null) thumbnailPath = getThumbnail(bufferedImage, hashes.md5) val imageSize = { (bufferedImage.getWidth, bufferedImage.getHeight) } val image = new Image(file.getAbsolutePath, thumbnailPath, imageSize, hashes) debug(s"Created image: $image") @@ -61,8 +60,9 @@ object ImageService extends Logging { def calculateThumbPath(md5:String):String = { //break the path down into 4 char parts val split:List[String] = md5.grouped(4).toList - var path:String = PropertiesService.get(PropertiesEnum.ThumbnailDirectory.toString) + PropertiesService.get(PropertiesEnum.ThumbnailSize.toString) + "/" - for (seg <- split) path += seg + "/" + var dirPath = "" + for (seg <- split) dirPath += seg + "/" + var path:String = s"${PropertiesService.get(PropertiesEnum.ThumbnailDirectory.toString)}${PropertiesService.get(PropertiesEnum.ThumbnailSize.toString)}/$dirPath" try { val dir = new File(path) if (!dir.exists()) dir.mkdirs() @@ -76,13 +76,6 @@ object ImageService extends Logging { def lookupThumbnailPath(md5:String):String = { var thumbPath:String = null if (md5 != null) { - //get from memory cache if possible - try { - if (thumbnailCache.isKeyInCache(md5)) thumbPath = thumbnailCache.get(md5).getObjectValue.asInstanceOf[String] - } catch { - case npe:NullPointerException => error(s"Error grabbing \'$md5\' from cache. thumbPath: \'$thumbPath\'", npe) - } - //get from datastore if possible //check for the actual file val checkPath = calculateThumbPath(md5) if (new File(checkPath).exists) thumbPath = checkPath @@ -108,14 +101,6 @@ object ImageService extends Logging { path } - def saveThumbnail(md5:String, thumbnailPath:String):String = { - if (md5 == null || thumbnailPath == null) error(s"Error trying to save a md5: $md5 path: $thumbnailPath") - //save to cache - thumbnailCache.put(new Element(md5, thumbnailPath)) - //save to datastore - thumbnailPath - } - /** * Get the raw data for an image */ diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf index 5d0a916..70d7bb1 100644 --- a/src/test/resources/application.conf +++ b/src/test/resources/application.conf @@ -8,48 +8,53 @@ akka { //Default App Settings app { - version.current = "${build-version}" - timed = false - engine { - //Concurrency Settings - concurrent { - similarity.limit = 2 - processing.limit = 2 - } + version.current = "${build-version}" + timed = false + engine { + //Concurrency Settings + concurrent { + similarity.limit = 2 + processing.limit = 2 } - #Default Image Settings - image { - //images must be 90% similar - differenceThreshold = 0.90 - //control generation of hashes for new images. - hash { - precision=64 - } - ahash { - use = true - weight = 0.70 - precision = 8 - tolerence = 8 - } - dhash { - use = true - weight = 0.85 - precision = 8 - tolerence = 8 - } - phash { - //set to false if hashing images is taking too long - use = true - weight = 1.0 - precision = 32 - tolerence = 8 - } + } + #Default Image Settings + image { + //images must be 90% similar + differenceThreshold = 0.90 + //control generation of hashes for new images. + hash { + precision=64 } - //Default Thumbnail Settings - thumbnail { - //Directory where to store thumbnails - directory = ".cache/thumbnails/" - //Size of the thumbnail to generate and store - size = 128 + ahash { + use = true + weight = 0.70 + precision = 8 + tolerence = 8 } + dhash { + use = true + weight = 0.85 + precision = 8 + tolerence = 8 + } + phash { + //set to false if hashing images is taking too long + use = true + weight = 1.0 + precision = 32 + tolerence = 8 + } + } + //Default Thumbnail Settings + thumbnail { + //Directory where to store thumbnails + directory = ".cache/thumbnails/" + //Size of the thumbnail to generate and store + size = 128 + } + //Default Database Settings + database { + location = ".cache/database.db" + inMemory = false + } } \ No newline at end of file diff --git a/todo b/todo index a160966..c7718c2 100644 --- a/todo +++ b/todo @@ -6,4 +6,11 @@ Add functionality to ImageService -Generate thumbnails Improve functionality of the PropertiesService -Getters and Setters for sepcific values with proper cascading and type awareness - -Nicer debugging? \ No newline at end of file + -Nicer debugging? +Hibernate - H2 intergration + - setup hibernate + - setup h2 + - setup C3P0 connection pool + - transfer configuration files + - autoload configuration files? + - map DTO's \ No newline at end of file