Integration of the visual validation feature.

core/src/org/testar/ProtocolUtil.java

@@ -245,10 +245,20 @@ public static String getStateshot(State state){
 	 */
 	public static AWTCanvas getStateshotBinary(State state) {
 		Shape viewPort = null;
-		if (state.childCount() > 0){
-			viewPort = state.child(0).get(Tags.Shape, null);
-			if (viewPort != null && (viewPort.width() * viewPort.height() < 1))
-				viewPort = null;
+		for (int index = 0; index < state.childCount(); index++){
+			// While testing Word (2109 build 14430.20270) we noticed that the height of the screenshots was only 5px.
+			// After investigation, we noticed that root contained 5 children. Four had "MSO_BORDEREFFECT_WINDOW_CLASS"
+			// and had none children. And only one was called "OpusApp" and contained child elements.
+			// Ideally this check is more strict; (frameworkId == Win32 and classname != MSO_BORDEREFFECT_WINDOW_CLASS)
+			// but unfortunately these tags are not available at this generic level. Previous implementation directly
+			// used the first child. However, it makes more sense to select the widget which contains children.
+			if (state.child(index).childCount() != 0) {
+				viewPort = state.child(index).get(Tags.Shape, null);
+				if (viewPort != null && (viewPort.width() * viewPort.height() < 1)) {
+					viewPort = null;
+				}
+				break;
+			}
 		}
 
 		//If the state Shape is not properly obtained, or the State has an error, use full monitor screen
@@ -260,8 +270,8 @@ public static AWTCanvas getStateshotBinary(State state) {
 		AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(viewPort.x(), viewPort.y(), viewPort.width(), viewPort.height()), getRootWindowHandle(state), AWTCanvas.StorageFormat.PNG, 1);
 		return scrshot;
 	}
-	
-	public static String getActionshot(State state, Action action){
+
+	public static AWTCanvas getActionshot(State state, Action action){
 		List<Finder> targets = action.get(Tags.Targets, null);
 		if (targets != null){
 			Widget w;
@@ -274,14 +284,14 @@ public static String getActionshot(State state, Action action){
 				r = new Rectangle((int)s.x(), (int)s.y(), (int)s.width(), (int)s.height());
 				actionArea = actionArea.union(r);
 			}
-			if (actionArea.isEmpty())
+			if (actionArea.isEmpty()) {
 				return null;
-			AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state),
+			}
+			return AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state),
 														 AWTCanvas.StorageFormat.PNG, 1);
-			return ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), scrshot);
 		}
 		return null;
-	}	
+	}
 
 	private static long getRootWindowHandle(State state) {
 		long windowHandle = 0;

core/src/org/testar/monkey/Util.java

@@ -545,7 +545,7 @@ public static String readFile(File path) {
 	}
 
 	public static File createTempDir() {
-		return createTempDir("org.fruit.", Long.toString(System.nanoTime()));
+		return createTempDir("org.testar.monkey.", Long.toString(System.nanoTime()));
 	}
 
 	public static File createTempDir(String pref, String suff) {
@@ -562,11 +562,11 @@ public static File createTempDir(String pref, String suff) {
 	}
 
 	public static File createTempFile() {
-		return createTempFile("org.fruit.", Long.toString(System.nanoTime()), null);
+		return createTempFile("org.testar.monkey.", Long.toString(System.nanoTime()), null);
 	}
 
 	public static File createTempFile(String content) {
-		return createTempFile("org.fruit.", Long.toString(System.nanoTime()), content);
+		return createTempFile("org.testar.monkey.", Long.toString(System.nanoTime()), content);
 	}
 
 	public static File createTempFile(String pref, String suff, String content) {

core/src/org/testar/monkey/alayer/AWTCanvas.java

@@ -45,11 +45,12 @@
 import java.awt.geom.AffineTransform;
 import java.awt.geom.Rectangle2D;
 import java.awt.image.BufferedImage;
+import java.awt.image.ColorModel;
 import java.awt.image.DataBuffer;
 import java.awt.image.DataBufferInt;
+import java.awt.image.WritableRaster;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
-import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -74,7 +75,7 @@
 
 public class AWTCanvas implements Image, Canvas {
 
-	public static enum StorageFormat{ JPEG, PNG, BMP; }
+	public enum StorageFormat{ JPEG, PNG, BMP }
 
 	public static void saveAsJpeg(BufferedImage image, OutputStream os, double quality) throws IOException{
 		if(quality == 1){
@@ -131,12 +132,8 @@ public static AWTCanvas fromScreenshot(Rect r, long windowHandle, StorageFormat
 	}
 
 	public static AWTCanvas fromFile(String file) throws IOException{
-		BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(file)));
-
-		try{
+		try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
 			return fromInputStream(bis);
-		}finally{
-			bis.close();
 		}
 	}
 
@@ -154,9 +151,9 @@ public static AWTCanvas fromInputStream(InputStream is) throws IOException{
 
 	private static final long serialVersionUID = -5041497503329308870L;
 	protected transient BufferedImage img;
-	private StorageFormat format;
-	private double quality;
-	private double x, y;
+	private final StorageFormat format;
+	private final double quality;
+	private final double x, y;
 	transient Graphics2D gr;
 	static final Pen defaultPen = Pen.PEN_DEFAULT;
 	double fontSize, strokeWidth;
@@ -188,11 +185,8 @@ public AWTCanvas(double x, double y, BufferedImage image, StorageFormat format,
 		this.format = format;
 		this.quality = quality;
 		gr = img.createGraphics();
-		//		gr.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING,
-		//				RenderingHints.VALUE_TEXT_ANTIALIAS_ON);
-
-		adjustPen(defaultPen);		
-		//gr.setComposite(AlphaComposite.Clear);
+
+		adjustPen(defaultPen);
 	}
 
 	public void begin() {}
@@ -203,7 +197,16 @@ public void end() {}
 	public double x(){ return x; }
 	public double y(){ return y; }
 	public BufferedImage image(){ return img; }
-
+
+	/**
+	 * @return A deep copy of the image.
+	 */
+	public BufferedImage deepCopyImage() {
+		ColorModel cm = img.getColorModel();
+		WritableRaster raster = img.copyData(img.getRaster().createCompatibleWritableRaster());
+		return new BufferedImage(cm, raster, cm.isAlphaPremultiplied(), null);
+	}
+
 	private void adjustPen(Pen pen){
 		Double tstrokeWidth = pen.strokeWidth();
 		if(tstrokeWidth == null)
@@ -221,7 +224,7 @@ private void adjustPen(Pen pen){
 			strokePattern = tstrokePattern;
 			strokeWidth = tstrokeWidth;
 			strokeCaps = tstrokeCaps;
-			gr.setStroke(new BasicStroke((float)(double)strokeWidth));
+			gr.setStroke(new BasicStroke((float)strokeWidth));
 		}
 
 		Color tcolor = pen.color();
@@ -244,7 +247,7 @@ private void adjustPen(Pen pen){
 		if(!tfont.equals(font) || !tfontSize.equals(fontSize)){
 			font = tfont;
 			fontSize = tfontSize;
-			gr.setFont(new Font(font, Font.PLAIN, (int)(double)fontSize));
+			gr.setFont(new Font(font, Font.PLAIN, (int)fontSize));
 		}
 
 		FillPattern tfillPattern = pen.fillPattern();
@@ -328,26 +331,18 @@ public void saveAsJpeg(OutputStream os, double quality) throws IOException{
 	}
 
 	public void saveAsJpeg(String file, double quality) throws IOException{
-		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file)));
-
-		try{
+		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
 			saveAsJpeg(bos, quality);
-		}finally{
-			bos.close();
 		}
 	}
 
 	public void saveAsPng(OutputStream os) throws IOException{
 		saveAsPng(img, os);
 	}
 
-	public void saveAsPng(String file) throws IOException{		
-		BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file)));
-
-		try{
+	public void saveAsPng(String file) throws IOException{
+		try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
 			saveAsPng(bos);
-		}finally{
-			bos.close();
 		}
 	}
 
@@ -375,7 +370,7 @@ public void paint(Canvas canvas, double x, double y, double width,
 			double height) {
 		Assert.notNull(canvas);
 
-		int data[] = ((DataBufferInt)img.getRaster().getDataBuffer()).getData();
+		int[] data = ((DataBufferInt)img.getRaster().getDataBuffer()).getData();
 		canvas.image(canvas.defaultPen(), x, y, width, height,
 				data, img.getWidth(), img.getHeight());
 	}
@@ -392,7 +387,7 @@ public void paint(Canvas canvas, Rect srcRect, Rect destRect) {
 		BufferedImage subImage = new BufferedImage(srcWidth, srcHeight, BufferedImage.TYPE_INT_ARGB);
 		subImage.getGraphics().drawImage(img.getSubimage(srcX, srcY, srcWidth, srcHeight), 0, 0, srcWidth, srcHeight, null);
 
-		int area[] = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData();
+		int[] area = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData();
 		canvas.image(canvas.defaultPen(), destRect.x(), destRect.y(), destRect.width(), destRect.height(), area, srcWidth, srcHeight);
 	}
 
@@ -412,7 +407,7 @@ public void triangle(Pen pen, double x1, double y1, double x2, double y2,
 	 * @author urueda
 	 */
 	public float compareImage(AWTCanvas img) {
-		//long now = System.currentTimeMillis();		
+		//long now = System.currentTimeMillis();
 		DataBuffer dbThis = this.img.getData().getDataBuffer(),
 				   dbImg = img.img.getData().getDataBuffer();
 		int sizeThis = dbThis.getSize(),
@@ -435,10 +430,10 @@ else if (sizeThis < sizeImg)
 		float meanSize = (sizeThis + sizeImg) / 2;
 		float percent = sizeSimilarity - (1.0f - (equalPixels / meanSize));
 		//System.out.println("Image comparison took : " + (System.currentTimeMillis() - now) + " ms");
-		return (percent < 0f ? 0f : (percent > 1f ? 1f : percent));
+		return (percent < 0f ? 0f : Math.min(percent, 1f));
 	}	
 
 	public void release() {}
-	
+
 	public String toString(){ return "AWTCanvas (width: " + width() + " height: " + height() + ")";	}
 }

core/src/org/testar/serialisation/ScreenshotSerialiser.java

@@ -109,6 +109,8 @@ public void run(){
 					r.scrshot.saveAsPng(r.scrshotPath);
 				} catch (IOException e) {
 					LogSerialiser.log("I/O exception saving screenshot <" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical);
+				} catch (NullPointerException e){
+					LogSerialiser.log("Screenshot was empty" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical);
 				}
 			}
 		}

testar/build.gradle

@@ -116,6 +116,9 @@ dependencies {
     implementation jnativehook
     implementation 'com.google.guava:guava:26.0-jre'
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.8'
+    implementation group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.14.0'
+    implementation group: 'org.bytedeco', name: 'tesseract-platform', version: '4.1.1-1.5.4'
+    implementation group: 'org.bytedeco', name: 'javacv', version: '1.5.5'
     implementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '9.4.30.v20200611'
     implementation group: 'org.eclipse.jetty', name: 'jetty-servlet', version: '9.4.30.v20200611'
     implementation group: 'org.eclipse.jetty', name: 'jetty-webapp', version: '9.4.30.v20200611'

testar/resources/log4j2.xml

@@ -18,7 +18,7 @@
             <AppenderRef ref="Console"/>
             <AppenderRef ref="RollingFile"/>
         </Logger>
-        <Logger name="nl.ou.testar" level="DEBUG" additivity="false">
+        <Logger name="org.testar" level="DEBUG" additivity="false">
             <AppenderRef ref="Console"/>
             <AppenderRef ref="RollingFile"/>
         </Logger>
@@ -28,4 +28,4 @@
             <AppenderRef ref="RollingFile"/>
         </Root>
     </Loggers>
-</Configuration>
+</Configuration>

testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java

@@ -34,8 +34,11 @@
 import java.util.Set;
 
 import org.testar.ProtocolUtil;
+import org.testar.monkey.alayer.Tags;
+import org.testar.serialisation.ScreenshotSerialiser;
 import org.testar.simplestategraph.GuiStateGraphWithVisitedActions;
 import org.testar.monkey.Util;
+import org.testar.monkey.alayer.AWTCanvas;
 import org.testar.monkey.alayer.Action;
 import org.testar.monkey.alayer.SUT;
 import org.testar.monkey.alayer.State;
@@ -161,7 +164,10 @@ protected boolean executeAction(SUT system, State state, Action action){
 			//System.out.println("DEBUG: action: "+action.toString());
 			//System.out.println("DEBUG: action short: "+action.toShortString());
 			if(action.toShortString().equalsIgnoreCase("LeftClickAt")){
-				String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action);
+				String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot(
+						state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						ProtocolUtil.getActionshot(state,action));
 				Eye eye = new Eye();
 				try {
 					//System.out.println("DEBUG: sikuli clicking ");
@@ -179,8 +185,10 @@ protected boolean executeAction(SUT system, State state, Action action){
 			}else if(action.toShortString().contains("ClickTypeInto(")){
 				String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")"));
 				//System.out.println("parsed text:"+textToType);
-				String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action);
-				Util.pause(halfWait);
+				String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot(
+						state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						ProtocolUtil.getActionshot(state,action));				Util.pause(halfWait);
 				Eye eye = new Eye();
 				try {
 					//System.out.println("DEBUG: sikuli typing ");

testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java

@@ -33,6 +33,8 @@
 import java.util.Set;
 
 import org.testar.ProtocolUtil;
+import org.testar.monkey.alayer.Tags;
+import org.testar.serialisation.ScreenshotSerialiser;
 import org.testar.simplestategraph.GuiStateGraphWithVisitedActions;
 import org.testar.monkey.Util;
 import org.testar.monkey.alayer.Action;
@@ -157,7 +159,10 @@ protected boolean executeAction(SUT system, State state, Action action){
 			//System.out.println("DEBUG: action: "+action.toString());
 			//System.out.println("DEBUG: action short: "+action.toShortString());
 			if(action.toShortString().equalsIgnoreCase("LeftClickAt")){
-				String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action);
+				String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot(
+						state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						ProtocolUtil.getActionshot(state,action));
 				Screen sikuliScreen = new Screen();
 				try {
 					//System.out.println("DEBUG: sikuli clicking ");
@@ -174,7 +179,10 @@ protected boolean executeAction(SUT system, State state, Action action){
 			}else if(action.toShortString().contains("ClickTypeInto(")){
 				String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")"));
 				//System.out.println("parsed text:"+textToType);
-				String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action);
+				String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot(
+						state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"),
+						ProtocolUtil.getActionshot(state,action));
 				Util.pause(halfWait);
 				Screen sikuliScreen = new Screen();
 				try {

testar/src/org/testar/Logger.java

@@ -0,0 +1,17 @@
+package org.testar;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+
+/**
+ * Facade for the log4j logger.
+ * Reduces the need to initialize the logger in every class.
+ * By wrapping {@code log4j.logger.log} with a tag argument a more uniformed log is realized.
+ */
+public class Logger {
+    private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger();
+
+    public static void log(Level level, String tag, String message, Object... params) {
+        LOGGER.log(level, "[" + tag + "] " + message, params);
+    }
+}

testar/src/org/testar/OutputStructure.java

@@ -97,9 +97,10 @@ public static void createOutputSUTname(Settings settings) {
 					executedSUTname = domain;
 				}
 				else if (sutConnectorValue.contains(".exe")) {
-					int startSUT = sutConnectorValue.lastIndexOf(File.separator)+1;
 					int endSUT = sutConnectorValue.indexOf(".exe");
-					String sutName = sutConnectorValue.substring(startSUT, endSUT);
+					String pathWithoutArguments = sutConnectorValue.substring(0, endSUT);
+					int startSUT = pathWithoutArguments.lastIndexOf(File.separator)+1;
+					String sutName = pathWithoutArguments.substring(startSUT, endSUT);
 					executedSUTname = sutName;
 				}
 				else if (sutConnectorValue.contains(".jar")) {

testar/src/org/testar/extendedsettings/ExampleSetting.java

@@ -47,7 +47,7 @@ public static ExampleSetting CreateDefault() {
 
     @Override
     public int compareTo(ExampleSetting other) {
-        if (test.contentEquals(other.test)){
+        if (test.contentEquals(other.test)) {
             return 0;
         }
         return -1;