Browse Source

Fixed broken unit test. Added some properties and functions. Added a timing trait and benchmarking to DHash tests

master
Drew Short 11 years ago
parent
commit
4e9c5cf87f
  1. 2
      .gitignore
  2. 15
      src/includes/log4j.properties
  3. BIN
      src/includes/sample/sample_01_medium.jpg
  4. BIN
      src/includes/sample/sample_01_small.jpg
  5. 5
      src/main/java/com/sothr/imagetools/App.java
  6. 2
      src/main/java/com/sothr/imagetools/AppCLI.java
  7. 55
      src/main/java/com/sothr/imagetools/AppConfig.java
  8. 6
      src/main/resources/default.properties
  9. 8
      src/main/scala/com/sothr/imagetools/hash/DHash.scala
  10. 6
      src/main/scala/com/sothr/imagetools/hash/HashService.scala
  11. 68
      src/main/scala/com/sothr/imagetools/image/ImageService.scala
  12. 5
      src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala
  13. 13
      src/main/scala/com/sothr/imagetools/util/PropertiesService.scala
  14. 31
      src/main/scala/com/sothr/imagetools/util/Timing.scala
  15. 6
      src/test/resources/default.properties
  16. 7
      src/test/scala/com/sothr/imagetools/BaseTest.scala
  17. 7
      src/test/scala/com/sothr/imagetools/TestParams.scala
  18. 69
      src/test/scala/com/sothr/imagetools/hash/HashServiceTest.scala

2
.gitignore

@ -29,4 +29,6 @@ name.info
version.info
# Image Tools Log Files
*.debug
*.info
*.err

15
src/includes/log4j.properties

@ -1,13 +1,22 @@
log4j.rootLogger=DEBUG, C, IL, EL
log4j.rootLogger=DEBUG, C, DL, IL, EL
# Console Output
log4j.appender.C=org.apache.log4j.ConsoleAppender
log4j.appender.C.layout=org.apache.log4j.EnhancedPatternLayout
log4j.appender.C.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n
# Debug Rolling Log
log4j.appender.DL=org.apache.log4j.RollingFileAppender
log4j.appender.DL.Threshold=DEBUG
log4j.appender.DL.File=Image-Tools.debug
log4j.appender.DL.MaxFileSize=500KB
log4j.appender.DL.MaxBackupIndex=1
log4j.appender.DL.layout=org.apache.log4j.EnhancedPatternLayout
log4j.appender.DL.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n
# Info Rolling Log
log4j.appender.IL.Threshold=INFO
log4j.appender.IL=org.apache.log4j.RollingFileAppender
log4j.appender.IL.Threshold=INFO
log4j.appender.IL.File=Image-Tools.info
log4j.appender.IL.MaxFileSize=100KB
log4j.appender.IL.MaxBackupIndex=1
@ -15,8 +24,8 @@ log4j.appender.IL.layout=org.apache.log4j.EnhancedPatternLayout
log4j.appender.IL.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n
# Error Rolling Log
log4j.appender.EL.Threshold=ERROR
log4j.appender.EL=org.apache.log4j.RollingFileAppender
log4j.appender.EL.Threshold=ERROR
log4j.appender.EL.File=Image-Tools.err
log4j.appender.EL.MaxFileSize=100KB
log4j.appender.EL.MaxBackupIndex=1

BIN
src/includes/sample/sample_01_medium.jpg

After

Width: 1824  |  Height: 1368  |  Size: 1.5 MiB

BIN
src/includes/sample/sample_01_small.jpg

After

Width: 912  |  Height: 684  |  Size: 519 KiB

5
src/main/java/com/sothr/imagetools/App.java

@ -25,7 +25,7 @@ public class App extends Application
public static void main( String[] args )
{
AppConfig.configLogging();
AppConfig.configureApp();
try {
//try to run the UI
launch(args);
@ -37,8 +37,7 @@ public class App extends Application
@Override
public void init() throws Exception{
AppConfig.configLogging();
AppConfig.loadProperties();
AppConfig.configureApp();
logger = LoggerFactory.getLogger(this.getClass());
logger.info("Initializing Image Tools");
List<String> parameters = this.getParameters().getRaw();

2
src/main/java/com/sothr/imagetools/AppCLI.java

@ -11,7 +11,7 @@ class AppCLI {
private static Logger logger;
public static void main(String[] args) {
AppConfig.configLogging();
AppConfig.configureApp();
logger = LoggerFactory.getLogger(AppCLI.class);
logger.info("Started Image Tools CLI");
System.out.println("Hello World");

55
src/main/java/com/sothr/imagetools/AppConfig.java

@ -1,7 +1,9 @@
package com.sothr.imagetools;
import com.sothr.imagetools.util.PropertiesService;
import com.sothr.imagetools.util.PropertiesEnum;
import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.BasicConfigurator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -21,8 +23,20 @@ class AppConfig {
private static final String USERPROPERTIESFILE = "./config.xml";
private static Boolean loadedProperties = false;
public static void configureApp() {
//configSimpleLogging();
loadProperties();
configLogging();
}
public static void configSimpleLogging() {
BasicConfigurator.configure();
}
public static void configLogging(String location) {
//Logging Config
//remove previous configuration if it exists
//BasicConfigurator.resetConfiguration();
File file = new File(location);
Boolean fromFile = false;
if (file.exists()) {
@ -31,14 +45,41 @@ class AppConfig {
} else {
//Simple error logging configuration
Properties defaultProps = new Properties();
defaultProps.setProperty("log4j.rootLogger","ERROR, A1");
String rootLogger = "DEBUG";
if (Boolean.valueOf(PropertiesService.get(PropertiesEnum.LogDebug().toString()))) {
//Rolling Debug logger
rootLogger += ", DL";
defaultProps.setProperty("log4j.appender.DL","org.apache.log4j.RollingFileAppender");
defaultProps.setProperty("log4j.appender.DL.Threshold","DEBUG");
defaultProps.setProperty("log4j.appender.DL.File","Image-Tools.debug");
defaultProps.setProperty("log4j.appender.DL.MaxFileSize","500KB");
defaultProps.setProperty("log4j.appender.DL.MaxBackupIndex","1");
defaultProps.setProperty("log4j.appender.DL.layout","org.apache.log4j.EnhancedPatternLayout");
defaultProps.setProperty("log4j.appender.DL.layout.ConversionPattern","%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n");
}
if (Boolean.valueOf(PropertiesService.get(PropertiesEnum.LogInfo().toString()))) {
//Rolling Info logger
rootLogger += ", IL";
defaultProps.setProperty("log4j.appender.IL","org.apache.log4j.RollingFileAppender");
defaultProps.setProperty("log4j.appender.IL.Threshold","INFO");
defaultProps.setProperty("log4j.appender.IL.File","Image-Tools.info");
defaultProps.setProperty("log4j.appender.IL.MaxFileSize","100KB");
defaultProps.setProperty("log4j.appender.IL.MaxBackupIndex","1");
defaultProps.setProperty("log4j.appender.IL.layout","org.apache.log4j.EnhancedPatternLayout");
defaultProps.setProperty("log4j.appender.IL.layout.ConversionPattern","%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n");
}
if (Boolean.valueOf(PropertiesService.get(PropertiesEnum.LogError().toString()))) {
//Rolling Error logger
defaultProps.setProperty("log4j.appender.A1","org.apache.log4j.RollingFileAppender");
defaultProps.setProperty("log4j.appender.A1.File","Image-Tools.err");
defaultProps.setProperty("log4j.appender.A1.MaxFileSize","100KB");
defaultProps.setProperty("log4j.appender.A1.MaxBackupIndex","1");
defaultProps.setProperty("log4j.appender.A1.layout","org.apache.log4j.EnhancedPatternLayout");
defaultProps.setProperty("log4j.appender.A1.layout.ConversionPattern","%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n");
rootLogger += ", EL";
defaultProps.setProperty("log4j.appender.EL","org.apache.log4j.RollingFileAppender");
defaultProps.setProperty("log4j.appender.EL.Threshold","ERROR");
defaultProps.setProperty("log4j.appender.EL.File","Image-Tools.err");
defaultProps.setProperty("log4j.appender.EL.MaxFileSize","100KB");
defaultProps.setProperty("log4j.appender.EL.MaxBackupIndex","1");
defaultProps.setProperty("log4j.appender.EL.layout","org.apache.log4j.EnhancedPatternLayout");
defaultProps.setProperty("log4j.appender.EL.layout.ConversionPattern","%d{yy-MM-dd HH:mm:ss} %-5p [%c{3.}] - %m%n");
}
defaultProps.setProperty("log4j.rootLogger",rootLogger);
PropertyConfigurator.configure(defaultProps);
}
logger = LoggerFactory.getLogger(AppConfig.class);

6
src/main/resources/default.properties

@ -2,6 +2,12 @@
#Image Tools version: ${project.version}
version=${project.version}
#Default App Settings
app.timed=false
app.log.debug=false
app.log.info=false
app.log.error=true
#Default Image Settings
#images must be 90% similar
image.differenceThreshold=0.90

8
src/main/scala/com/sothr/imagetools/hash/DHash.scala

@ -1,12 +1,16 @@
package com.sothr.imagetools.hash
import grizzled.slf4j.Logging
/**
* Created by dev on 1/22/14.
*/
object DHash extends PerceptualHasher {
object DHash extends PerceptualHasher with Logging {
def getHash(imageData: Array[Array[Int]]): Long = {
debug("Generating DHash")
val width = imageData.length
val height = imageData(0).length
debug(s"Image data size: ${width}x${height}")
var hash = 0L
for (row <- 0 until width) {
//println(f"Row: $row%d")
@ -15,12 +19,14 @@ object DHash extends PerceptualHasher {
//process each column
for (col <- 0 until height) {
debug(s"previousPixel: $previousPixel previousLocation: $previousLocation")
//println(f"Column: $col%d")
hash <<= 1
val pixel = imageData(row)(col)
//binary or the current bit based on whether the value
//of the current pixel is greater or equal to the previous pixel
if (pixel >= previousPixel) hash |= 1 else hash |= 0
debug(s"hash: hash")
previousPixel = pixel
previousLocation = (row, col)
}

6
src/main/scala/com/sothr/imagetools/hash/HashService.scala

@ -43,7 +43,7 @@ object HashService extends Logging {
}
def getAhash(image:BufferedImage):Long = {
debug("Generating an AHash")
debug("Started generating an AHash")
val grayImage = ImageService.convertToGray(image)
val resizedImage = ImageService.resize(grayImage, PropertiesService.get(PropertiesEnum.AhashPrecision.toString).toInt, true)
val imageData = ImageService.getImageData(resizedImage)
@ -51,7 +51,7 @@ object HashService extends Logging {
}
def getDhash(image:BufferedImage):Long = {
debug("Generating an DHash")
debug("Started generating an DHash")
val grayImage = ImageService.convertToGray(image)
val resizedImage = ImageService.resize(grayImage, PropertiesService.get(PropertiesEnum.DhashPrecision.toString).toInt, true)
val imageData = ImageService.getImageData(resizedImage)
@ -59,7 +59,7 @@ object HashService extends Logging {
}
def getPhash(image:BufferedImage):Long = {
debug("Generating an PHash")
debug("Started generating an PHash")
val grayImage = ImageService.convertToGray(image)
val resizedImage = ImageService.resize(grayImage, PropertiesService.get(PropertiesEnum.PhashPrecision.toString).toInt, true)
val imageData = ImageService.getImageData(resizedImage)

68
src/main/scala/com/sothr/imagetools/image/ImageService.scala

@ -1,7 +1,7 @@
package com.sothr.imagetools
import grizzled.slf4j.Logging
import java.awt.image.{DataBufferByte, BufferedImage}
import java.awt.image.{DataBufferByte, BufferedImage, ColorConvertOp}
import net.coobird.thumbnailator.Thumbnails
object ImageService extends Logging {
@ -20,14 +20,25 @@ object ImageService extends Logging {
* @return
*/
def convertToGray(image:BufferedImage):BufferedImage = {
debug("Converting an image to grayscale")
val grayImage = new BufferedImage(image.getWidth, image.getHeight, BufferedImage.TYPE_BYTE_GRAY)
val g = image.getGraphics
g.drawImage(image,0,0,null)
g.dispose()
grayImage
//create a color conversion operation
val op = new ColorConvertOp(
image.getColorModel.getColorSpace,
grayImage.getColorModel.getColorSpace, null)
//convert the image to grey
val result = op.filter(image, grayImage)
//val g = image.getGraphics
//g.drawImage(image,0,0,null)
//g.dispose()
result
}
def resize(image:BufferedImage, size:Int, forced:Boolean=false):BufferedImage = {
debug(s"Resizing an image to size: ${size}x${size} forced: $forced")
if (forced) {
Thumbnails.of(image).forceSize(size,size).asBufferedImage
} else {
@ -44,21 +55,45 @@ object ImageService extends Logging {
private def convertTo2DWithoutUsingGetRGB(image:BufferedImage):Array[Array[Int]] = {
val pixels = image.getRaster.getDataBuffer.asInstanceOf[DataBufferByte].getData
val numPixels = pixels.length
val width = image.getWidth
val height = image.getHeight
val isSingleChannel = if(numPixels == (width * height)) true else false
val hasAlphaChannel = image.getAlphaRaster != null
debug(s"Converting image to 2d. width:$width height:$height")
val result = Array.ofDim[Int](height,width)
if (hasAlphaChannel) {
if (isSingleChannel) {
debug(s"Processing Single Channel Image")
val pixelLength = 1
var row = 0
var col = 0
debug(s"Processing pixels 0 until $numPixels by $pixelLength")
for (pixel <- 0 until numPixels by pixelLength) {
debug(s"Processing pixel: $pixel/${numPixels - 1}")
val argb:Int = pixels(pixel).toInt //singleChannel
debug(s"Pixel data: $argb")
result(row)(col) = argb
col += 1
if (col == width) {
col = 0
row += 1
}
}
}
else if (hasAlphaChannel) {
debug(s"Processing Four Channel Image")
val pixelLength = 4
var row = 0
var col = 0
for (pixel <- 0 until pixels.length by pixelLength) {
debug(s"Processing pixels 0 until $numPixels by $pixelLength")
for (pixel <- 0 until numPixels by pixelLength) {
debug(s"Processing pixel: $pixel/${numPixels - 1}")
var argb:Int = 0
argb += (pixels(pixel) & 0xff) << 24 //alpha
argb += (pixels(pixel + 1) & 0xff) //blue
argb += (pixels(pixel + 2) & 0xff) << 8 //green
argb += (pixels(pixel + 3) & 0xff) << 16 //red
argb += pixels(pixel).toInt << 24 //alpha
argb += pixels(pixel + 1).toInt //blue
argb += pixels(pixel + 2).toInt << 8 //green
argb += pixels(pixel + 3).toInt << 16 //red
result(row)(col) = argb
col += 1
if (col == width) {
@ -67,15 +102,18 @@ object ImageService extends Logging {
}
}
} else {
debug(s"Processing Three Channel Image")
val pixelLength = 3
var row = 0
var col = 0
for (pixel <- 0 until pixels.length by pixelLength) {
debug(s"Processing pixels 0 until $numPixels by $pixelLength")
for (pixel <- 0 until numPixels by pixelLength) {
debug(s"Processing pixel: $pixel/${numPixels - 1}")
var argb:Int = 0
argb += -16777216; // 255 alpha
argb += (pixels(pixel) & 0xff) //blue
argb += (pixels(pixel + 1) & 0xff) << 8 //green
argb += (pixels(pixel + 2) & 0xff) << 16 //red
argb += pixels(pixel).toInt //blue
argb += pixels(pixel + 1).toInt << 8 //green
argb += pixels(pixel + 2).toInt << 16 //red
result(row)(col) = argb
col += 1
if (col == width) {

5
src/main/scala/com/sothr/imagetools/util/PropertiesEnum.scala

@ -3,6 +3,11 @@ package com.sothr.imagetools.util
object PropertiesEnum extends Enumeration {
type PropertiesEnum = Value
val Version = Value("version")
//default app settings
val LogDebug = Value("app.log.debug")
val LogInfo = Value("app.log.info")
val LogError = Value("app.log.error")
val Timed = Value("app.timed")
//default image settings
val ImageDifferenceThreshold = Value("image.differenceThreshold")
val HashPrecision = Value("image.hash.precision")

13
src/main/scala/com/sothr/imagetools/util/PropertiesService.scala

@ -34,6 +34,13 @@ object PropertiesService extends Logging {
}
version = new Version(properties.getProperty("version"));
info(s"Detected Version: $version")
//load special properties
DebugLogEnabled = get(PropertiesEnum.LogDebug.toString).toBoolean
InfoLogEnabled = get(PropertiesEnum.LogInfo.toString).toBoolean
ErrorLogEnabled = get(PropertiesEnum.LogError.toString).toBoolean
TimingEnabled = get(PropertiesEnum.Timed.toString).toBoolean
info("Loaded Special Properties")
}
/**
@ -67,4 +74,10 @@ object PropertiesService extends Logging {
properties.setProperty(key, value)
}
//specific highly used properties
var DebugLogEnabled:Boolean = false
var InfoLogEnabled:Boolean = false
var ErrorLogEnabled:Boolean = false
var TimingEnabled:Boolean = false
}

31
src/main/scala/com/sothr/imagetools/util/Timing.scala

@ -0,0 +1,31 @@
package com.sothr.imagetools.util
import grizzled.slf4j.Logging
trait Timing extends Logging{
def time[R](block: => R): R = {
val t0 = System.currentTimeMillis
val result = block // call-by-name
val t1 = System.currentTimeMillis
info("Elapsed time: " + (t1 - t0) + "ms")
result
}
def getTime[R](block: => R):Long = {
val t0 = System.currentTimeMillis
val result = block // call-by-name
val t1 = System.currentTimeMillis
info("Elapsed time: " + (t1 - t0) + "ms")
(t1 - t0)
}
def getMean(times:Long*):Long = {
var ag = 0L
for (i <- times.indices) {
ag += times(i)
}
(ag / times.length)
}
}

6
src/test/resources/default.properties

@ -2,6 +2,12 @@
#Image Tools version: ${project.version}
version=${project.version}
#Default App Settings
app.timed=true
app.log.debug=true
app.log.info=true
app.log.error=true
#Default Image Settings
#images must be 90% similar
image.differenceThreshold=0.90

7
src/test/scala/com/sothr/imagetools/BaseTest.scala

@ -1,12 +1,13 @@
package com.sothr.imagetools
import grizzled.slf4j.Logging
import com.sothr.imagetools.util.Timing
import org.scalatest.{FunSuite,Matchers,OptionValues,Inside,Inspectors,BeforeAndAfter}
abstract class BaseTest extends FunSuite with Matchers with OptionValues with Inside with Inspectors with BeforeAndAfter {
abstract class BaseTest extends FunSuite with Matchers with OptionValues with Inside with Inspectors with BeforeAndAfter with Logging with Timing {
before {
AppConfig.configLogging()
AppConfig.loadProperties()
AppConfig.configureApp()
}
}

7
src/test/scala/com/sothr/imagetools/TestParams.scala

@ -0,0 +1,7 @@
package com.sothr.imagetools
object TestParams {
val LargeSampleImage1 = "target/sample/sample_01_large.jpg"
val MediumSampleImage1 = "target/sample/sample_01_medium.jpg"
val SmallSampleImage1 = "target/sample/sample_01_small.jpg"
}

69
src/test/scala/com/sothr/imagetools/hash/HashServiceTest.scala

@ -1,6 +1,6 @@
package com.sothr.imagetools.hash
import com.sothr.imagetools.BaseTest
import com.sothr.imagetools.{BaseTest, TestParams}
import javax.imageio.ImageIO
import java.io.File
@ -9,11 +9,72 @@ import java.io.File
*/
class HashServiceTest extends BaseTest {
test("Calculate DHash") {
val sample = new File("./target/sample/sample_01_large.jpg")
def benchmarkDHashTestCase(filePath:String):Long = {
val sample = new File(filePath)
val image = ImageIO.read(sample)
HashService.getDhash(image)
}
test("Benchmark DHash") {
info("Benchmarking DHash")
info("DHash Large Image 3684x2736")
val largeTime1 = getTime { benchmarkDHashTestCase(TestParams.LargeSampleImage1) }
val largeTime2 = getTime { benchmarkDHashTestCase(TestParams.LargeSampleImage1) }
val largeTime3 = getTime { benchmarkDHashTestCase(TestParams.LargeSampleImage1) }
val largeTime4 = getTime { benchmarkDHashTestCase(TestParams.LargeSampleImage1) }
val largeTime5 = getTime { benchmarkDHashTestCase(TestParams.LargeSampleImage1) }
val largeMean = getMean(largeTime1, largeTime2, largeTime3, largeTime4, largeTime5)
info(s"The mean time of 5 tests for large was: $largeMean ms")
info("DHash Medium Image 1824x1368")
val mediumTime1 = getTime { benchmarkDHashTestCase(TestParams.MediumSampleImage1) }
val mediumTime2 = getTime { benchmarkDHashTestCase(TestParams.MediumSampleImage1) }
val mediumTime3 = getTime { benchmarkDHashTestCase(TestParams.MediumSampleImage1) }
val mediumTime4 = getTime { benchmarkDHashTestCase(TestParams.MediumSampleImage1) }
val mediumTime5 = getTime { benchmarkDHashTestCase(TestParams.MediumSampleImage1) }
val mediumMean = getMean(mediumTime1, mediumTime2, mediumTime3, mediumTime4, mediumTime5)
info(s"The mean time of 5 tests for medium was: $mediumMean ms")
info("DHash Small Image 912x684")
val smallTime1 = getTime { benchmarkDHashTestCase(TestParams.SmallSampleImage1) }
val smallTime2 = getTime { benchmarkDHashTestCase(TestParams.SmallSampleImage1) }
val smallTime3 = getTime { benchmarkDHashTestCase(TestParams.SmallSampleImage1) }
val smallTime4 = getTime { benchmarkDHashTestCase(TestParams.SmallSampleImage1) }
val smallTime5 = getTime { benchmarkDHashTestCase(TestParams.SmallSampleImage1) }
val smallMean = getMean(smallTime1, smallTime2, smallTime3, smallTime4, smallTime5)
info(s"The mean time of 5 tests for small was: $smallMean ms")
assert(true)
}
test("Calculate DHash Large Sample Image 1") {
debug("Starting 'Calculate DHash Large Sample Image 1' test")
val sample = new File(TestParams.LargeSampleImage1)
debug(s"Testing File: ${sample.getAbsolutePath} exists: ${sample.exists}")
val image = ImageIO.read(sample)
debug(s"Image: width: ${image.getWidth} height: ${image.getHeight}")
val hash = HashService.getDhash(image)
debug(s"Testing that $hash = -5198308484644955238L")
assert(hash == -5198308484644955238L)
}
test("Calculate DHash Medium Sample Image 1") {
debug("Starting 'Calculate DHash Medium Sample Image 1' test")
val sample = new File(TestParams.MediumSampleImage1)
debug(s"Testing File: ${sample.getAbsolutePath} exists: ${sample.exists}")
val image = ImageIO.read(sample)
debug(s"Image: width: ${image.getWidth} height: ${image.getHeight}")
val hash = HashService.getDhash(image)
debug(s"Testing that $hash = -5198308484644955238L")
assert(hash == -5198308484644955238L)
}
test("Calculate DHash Small Sample Image 1") {
debug("Starting 'Calculate DHash Small Sample Image 1' test")
val sample = new File(TestParams.SmallSampleImage1)
debug(s"Testing File: ${sample.getAbsolutePath} exists: ${sample.exists}")
val image = ImageIO.read(sample)
debug(s"Image: width: ${image.getWidth} height: ${image.getHeight}")
val hash = HashService.getDhash(image)
assert(hash == 0L)
debug(s"Testing that $hash = -5198299688551933030L")
assert(hash == -5198299688551933030L)
}
}
Loading…
Cancel
Save