diff --git a/core/src/org/testar/ProtocolUtil.java b/core/src/org/testar/ProtocolUtil.java index 0eebcfafa..0d7ed3c89 100644 --- a/core/src/org/testar/ProtocolUtil.java +++ b/core/src/org/testar/ProtocolUtil.java @@ -245,10 +245,20 @@ public static String getStateshot(State state){ */ public static AWTCanvas getStateshotBinary(State state) { Shape viewPort = null; - if (state.childCount() > 0){ - viewPort = state.child(0).get(Tags.Shape, null); - if (viewPort != null && (viewPort.width() * viewPort.height() < 1)) - viewPort = null; + for (int index = 0; index < state.childCount(); index++){ + // While testing Word (2109 build 14430.20270) we noticed that the height of the screenshots was only 5px. + // After investigation, we noticed that root contained 5 children. Four had "MSO_BORDEREFFECT_WINDOW_CLASS" + // and had none children. And only one was called "OpusApp" and contained child elements. + // Ideally this check is more strict; (frameworkId == Win32 and classname != MSO_BORDEREFFECT_WINDOW_CLASS) + // but unfortunately these tags are not available at this generic level. Previous implementation directly + // used the first child. However, it makes more sense to select the widget which contains children. + if (state.child(index).childCount() != 0) { + viewPort = state.child(index).get(Tags.Shape, null); + if (viewPort != null && (viewPort.width() * viewPort.height() < 1)) { + viewPort = null; + } + break; + } } //If the state Shape is not properly obtained, or the State has an error, use full monitor screen @@ -260,8 +270,8 @@ public static AWTCanvas getStateshotBinary(State state) { AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(viewPort.x(), viewPort.y(), viewPort.width(), viewPort.height()), getRootWindowHandle(state), AWTCanvas.StorageFormat.PNG, 1); return scrshot; } - - public static String getActionshot(State state, Action action){ + + public static AWTCanvas getActionshot(State state, Action action){ List targets = action.get(Tags.Targets, null); if (targets != null){ Widget w; @@ -274,14 +284,14 @@ public static String getActionshot(State state, Action action){ r = new Rectangle((int)s.x(), (int)s.y(), (int)s.width(), (int)s.height()); actionArea = actionArea.union(r); } - if (actionArea.isEmpty()) + if (actionArea.isEmpty()) { return null; - AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state), + } + return AWTCanvas.fromScreenshot(Rect.from(actionArea.x, actionArea.y, actionArea.width, actionArea.height), getRootWindowHandle(state), AWTCanvas.StorageFormat.PNG, 1); - return ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), scrshot); } return null; - } + } private static long getRootWindowHandle(State state) { long windowHandle = 0; diff --git a/core/src/org/testar/monkey/Util.java b/core/src/org/testar/monkey/Util.java index 92107ee37..2747e0320 100644 --- a/core/src/org/testar/monkey/Util.java +++ b/core/src/org/testar/monkey/Util.java @@ -545,7 +545,7 @@ public static String readFile(File path) { } public static File createTempDir() { - return createTempDir("org.fruit.", Long.toString(System.nanoTime())); + return createTempDir("org.testar.monkey.", Long.toString(System.nanoTime())); } public static File createTempDir(String pref, String suff) { @@ -562,11 +562,11 @@ public static File createTempDir(String pref, String suff) { } public static File createTempFile() { - return createTempFile("org.fruit.", Long.toString(System.nanoTime()), null); + return createTempFile("org.testar.monkey.", Long.toString(System.nanoTime()), null); } public static File createTempFile(String content) { - return createTempFile("org.fruit.", Long.toString(System.nanoTime()), content); + return createTempFile("org.testar.monkey.", Long.toString(System.nanoTime()), content); } public static File createTempFile(String pref, String suff, String content) { diff --git a/core/src/org/testar/monkey/alayer/AWTCanvas.java b/core/src/org/testar/monkey/alayer/AWTCanvas.java index 157612358..f461e94ca 100644 --- a/core/src/org/testar/monkey/alayer/AWTCanvas.java +++ b/core/src/org/testar/monkey/alayer/AWTCanvas.java @@ -45,11 +45,12 @@ import java.awt.geom.AffineTransform; import java.awt.geom.Rectangle2D; import java.awt.image.BufferedImage; +import java.awt.image.ColorModel; import java.awt.image.DataBuffer; import java.awt.image.DataBufferInt; +import java.awt.image.WritableRaster; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; -import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -74,7 +75,7 @@ public class AWTCanvas implements Image, Canvas { - public static enum StorageFormat{ JPEG, PNG, BMP; } + public enum StorageFormat{ JPEG, PNG, BMP } public static void saveAsJpeg(BufferedImage image, OutputStream os, double quality) throws IOException{ if(quality == 1){ @@ -131,12 +132,8 @@ public static AWTCanvas fromScreenshot(Rect r, long windowHandle, StorageFormat } public static AWTCanvas fromFile(String file) throws IOException{ - BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(file))); - - try{ + try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) { return fromInputStream(bis); - }finally{ - bis.close(); } } @@ -154,9 +151,9 @@ public static AWTCanvas fromInputStream(InputStream is) throws IOException{ private static final long serialVersionUID = -5041497503329308870L; protected transient BufferedImage img; - private StorageFormat format; - private double quality; - private double x, y; + private final StorageFormat format; + private final double quality; + private final double x, y; transient Graphics2D gr; static final Pen defaultPen = Pen.PEN_DEFAULT; double fontSize, strokeWidth; @@ -188,11 +185,8 @@ public AWTCanvas(double x, double y, BufferedImage image, StorageFormat format, this.format = format; this.quality = quality; gr = img.createGraphics(); - // gr.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, - // RenderingHints.VALUE_TEXT_ANTIALIAS_ON); - - adjustPen(defaultPen); - //gr.setComposite(AlphaComposite.Clear); + + adjustPen(defaultPen); } public void begin() {} @@ -203,7 +197,16 @@ public void end() {} public double x(){ return x; } public double y(){ return y; } public BufferedImage image(){ return img; } - + + /** + * @return A deep copy of the image. + */ + public BufferedImage deepCopyImage() { + ColorModel cm = img.getColorModel(); + WritableRaster raster = img.copyData(img.getRaster().createCompatibleWritableRaster()); + return new BufferedImage(cm, raster, cm.isAlphaPremultiplied(), null); + } + private void adjustPen(Pen pen){ Double tstrokeWidth = pen.strokeWidth(); if(tstrokeWidth == null) @@ -221,7 +224,7 @@ private void adjustPen(Pen pen){ strokePattern = tstrokePattern; strokeWidth = tstrokeWidth; strokeCaps = tstrokeCaps; - gr.setStroke(new BasicStroke((float)(double)strokeWidth)); + gr.setStroke(new BasicStroke((float)strokeWidth)); } Color tcolor = pen.color(); @@ -244,7 +247,7 @@ private void adjustPen(Pen pen){ if(!tfont.equals(font) || !tfontSize.equals(fontSize)){ font = tfont; fontSize = tfontSize; - gr.setFont(new Font(font, Font.PLAIN, (int)(double)fontSize)); + gr.setFont(new Font(font, Font.PLAIN, (int)fontSize)); } FillPattern tfillPattern = pen.fillPattern(); @@ -328,12 +331,8 @@ public void saveAsJpeg(OutputStream os, double quality) throws IOException{ } public void saveAsJpeg(String file, double quality) throws IOException{ - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file))); - - try{ + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) { saveAsJpeg(bos, quality); - }finally{ - bos.close(); } } @@ -341,13 +340,9 @@ public void saveAsPng(OutputStream os) throws IOException{ saveAsPng(img, os); } - public void saveAsPng(String file) throws IOException{ - BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(file))); - - try{ + public void saveAsPng(String file) throws IOException{ + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) { saveAsPng(bos); - }finally{ - bos.close(); } } @@ -375,7 +370,7 @@ public void paint(Canvas canvas, double x, double y, double width, double height) { Assert.notNull(canvas); - int data[] = ((DataBufferInt)img.getRaster().getDataBuffer()).getData(); + int[] data = ((DataBufferInt)img.getRaster().getDataBuffer()).getData(); canvas.image(canvas.defaultPen(), x, y, width, height, data, img.getWidth(), img.getHeight()); } @@ -392,7 +387,7 @@ public void paint(Canvas canvas, Rect srcRect, Rect destRect) { BufferedImage subImage = new BufferedImage(srcWidth, srcHeight, BufferedImage.TYPE_INT_ARGB); subImage.getGraphics().drawImage(img.getSubimage(srcX, srcY, srcWidth, srcHeight), 0, 0, srcWidth, srcHeight, null); - int area[] = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData(); + int[] area = ((DataBufferInt)subImage.getRaster().getDataBuffer()).getData(); canvas.image(canvas.defaultPen(), destRect.x(), destRect.y(), destRect.width(), destRect.height(), area, srcWidth, srcHeight); } @@ -412,7 +407,7 @@ public void triangle(Pen pen, double x1, double y1, double x2, double y2, * @author urueda */ public float compareImage(AWTCanvas img) { - //long now = System.currentTimeMillis(); + //long now = System.currentTimeMillis(); DataBuffer dbThis = this.img.getData().getDataBuffer(), dbImg = img.img.getData().getDataBuffer(); int sizeThis = dbThis.getSize(), @@ -435,10 +430,10 @@ else if (sizeThis < sizeImg) float meanSize = (sizeThis + sizeImg) / 2; float percent = sizeSimilarity - (1.0f - (equalPixels / meanSize)); //System.out.println("Image comparison took : " + (System.currentTimeMillis() - now) + " ms"); - return (percent < 0f ? 0f : (percent > 1f ? 1f : percent)); + return (percent < 0f ? 0f : Math.min(percent, 1f)); } public void release() {} - + public String toString(){ return "AWTCanvas (width: " + width() + " height: " + height() + ")"; } } diff --git a/core/src/org/testar/serialisation/ScreenshotSerialiser.java b/core/src/org/testar/serialisation/ScreenshotSerialiser.java index 2ece8d5e9..e61df56d1 100644 --- a/core/src/org/testar/serialisation/ScreenshotSerialiser.java +++ b/core/src/org/testar/serialisation/ScreenshotSerialiser.java @@ -109,6 +109,8 @@ public void run(){ r.scrshot.saveAsPng(r.scrshotPath); } catch (IOException e) { LogSerialiser.log("I/O exception saving screenshot <" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical); + } catch (NullPointerException e){ + LogSerialiser.log("Screenshot was empty" + r.scrshotPath + ">\n", LogSerialiser.LogLevel.Critical); } } } diff --git a/testar/build.gradle b/testar/build.gradle index 338efc3ef..f73805624 100644 --- a/testar/build.gradle +++ b/testar/build.gradle @@ -116,6 +116,9 @@ dependencies { implementation jnativehook implementation 'com.google.guava:guava:26.0-jre' implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.8' + implementation group: 'org.seleniumhq.selenium', name: 'selenium-java', version: '3.14.0' + implementation group: 'org.bytedeco', name: 'tesseract-platform', version: '4.1.1-1.5.4' + implementation group: 'org.bytedeco', name: 'javacv', version: '1.5.5' implementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '9.4.30.v20200611' implementation group: 'org.eclipse.jetty', name: 'jetty-servlet', version: '9.4.30.v20200611' implementation group: 'org.eclipse.jetty', name: 'jetty-webapp', version: '9.4.30.v20200611' diff --git a/testar/resources/log4j2.xml b/testar/resources/log4j2.xml index 12a26a861..c3e35295a 100644 --- a/testar/resources/log4j2.xml +++ b/testar/resources/log4j2.xml @@ -18,7 +18,7 @@ - + @@ -28,4 +28,4 @@ - \ No newline at end of file + diff --git a/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java b/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java index 01c46d848..7c57c6208 100644 --- a/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java +++ b/testar/resources/settings/desktop_simple_stategraph_eye/Protocol_desktop_simple_stategraph_eye.java @@ -34,8 +34,11 @@ import java.util.Set; import org.testar.ProtocolUtil; +import org.testar.monkey.alayer.Tags; +import org.testar.serialisation.ScreenshotSerialiser; import org.testar.simplestategraph.GuiStateGraphWithVisitedActions; import org.testar.monkey.Util; +import org.testar.monkey.alayer.AWTCanvas; import org.testar.monkey.alayer.Action; import org.testar.monkey.alayer.SUT; import org.testar.monkey.alayer.State; @@ -161,7 +164,10 @@ protected boolean executeAction(SUT system, State state, Action action){ //System.out.println("DEBUG: action: "+action.toString()); //System.out.println("DEBUG: action short: "+action.toShortString()); if(action.toShortString().equalsIgnoreCase("LeftClickAt")){ - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Eye eye = new Eye(); try { //System.out.println("DEBUG: sikuli clicking "); @@ -179,8 +185,10 @@ protected boolean executeAction(SUT system, State state, Action action){ }else if(action.toShortString().contains("ClickTypeInto(")){ String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")")); //System.out.println("parsed text:"+textToType); - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); - Util.pause(halfWait); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Util.pause(halfWait); Eye eye = new Eye(); try { //System.out.println("DEBUG: sikuli typing "); diff --git a/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java b/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java index f9b28f946..2bb4e6a5c 100644 --- a/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java +++ b/testar/resources/settings/desktop_simple_stategraph_sikulix/Protocol_desktop_simple_stategraph_sikulix.java @@ -33,6 +33,8 @@ import java.util.Set; import org.testar.ProtocolUtil; +import org.testar.monkey.alayer.Tags; +import org.testar.serialisation.ScreenshotSerialiser; import org.testar.simplestategraph.GuiStateGraphWithVisitedActions; import org.testar.monkey.Util; import org.testar.monkey.alayer.Action; @@ -157,7 +159,10 @@ protected boolean executeAction(SUT system, State state, Action action){ //System.out.println("DEBUG: action: "+action.toString()); //System.out.println("DEBUG: action short: "+action.toShortString()); if(action.toShortString().equalsIgnoreCase("LeftClickAt")){ - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Screen sikuliScreen = new Screen(); try { //System.out.println("DEBUG: sikuli clicking "); @@ -174,7 +179,10 @@ protected boolean executeAction(SUT system, State state, Action action){ }else if(action.toShortString().contains("ClickTypeInto(")){ String textToType = action.toShortString().substring(action.toShortString().indexOf("("), action.toShortString().indexOf(")")); //System.out.println("parsed text:"+textToType); - String widgetScreenshotPath = ProtocolUtil.getActionshot(state,action); + String widgetScreenshotPath = ScreenshotSerialiser.saveActionshot( + state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + ProtocolUtil.getActionshot(state,action)); Util.pause(halfWait); Screen sikuliScreen = new Screen(); try { diff --git a/testar/src/org/testar/Logger.java b/testar/src/org/testar/Logger.java new file mode 100644 index 000000000..14adeb77d --- /dev/null +++ b/testar/src/org/testar/Logger.java @@ -0,0 +1,17 @@ +package org.testar; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; + +/** + * Facade for the log4j logger. + * Reduces the need to initialize the logger in every class. + * By wrapping {@code log4j.logger.log} with a tag argument a more uniformed log is realized. + */ +public class Logger { + private static final org.apache.logging.log4j.Logger LOGGER = LogManager.getLogger(); + + public static void log(Level level, String tag, String message, Object... params) { + LOGGER.log(level, "[" + tag + "] " + message, params); + } +} diff --git a/testar/src/org/testar/OutputStructure.java b/testar/src/org/testar/OutputStructure.java index 7059efc55..63cc5622e 100644 --- a/testar/src/org/testar/OutputStructure.java +++ b/testar/src/org/testar/OutputStructure.java @@ -97,9 +97,10 @@ public static void createOutputSUTname(Settings settings) { executedSUTname = domain; } else if (sutConnectorValue.contains(".exe")) { - int startSUT = sutConnectorValue.lastIndexOf(File.separator)+1; int endSUT = sutConnectorValue.indexOf(".exe"); - String sutName = sutConnectorValue.substring(startSUT, endSUT); + String pathWithoutArguments = sutConnectorValue.substring(0, endSUT); + int startSUT = pathWithoutArguments.lastIndexOf(File.separator)+1; + String sutName = pathWithoutArguments.substring(startSUT, endSUT); executedSUTname = sutName; } else if (sutConnectorValue.contains(".jar")) { diff --git a/testar/src/org/testar/extendedsettings/ExampleSetting.java b/testar/src/org/testar/extendedsettings/ExampleSetting.java index c34950d30..9e8a62e30 100644 --- a/testar/src/org/testar/extendedsettings/ExampleSetting.java +++ b/testar/src/org/testar/extendedsettings/ExampleSetting.java @@ -47,7 +47,7 @@ public static ExampleSetting CreateDefault() { @Override public int compareTo(ExampleSetting other) { - if (test.contentEquals(other.test)){ + if (test.contentEquals(other.test)) { return 0; } return -1; diff --git a/testar/src/org/testar/extendedsettings/ExtendedSettingBase.java b/testar/src/org/testar/extendedsettings/ExtendedSettingBase.java index 21c57e51e..ddb19af75 100644 --- a/testar/src/org/testar/extendedsettings/ExtendedSettingBase.java +++ b/testar/src/org/testar/extendedsettings/ExtendedSettingBase.java @@ -36,7 +36,7 @@ public abstract class ExtendedSettingBase extends Observable implements IExtendedSetting, Comparable, Serializable { /** - * Notify the {@link IExtendedSettingContainer} that the specialization of this class needs to be saved. + * Notify the {@link ExtendedSettingContainer} that the specialization of this class needs to be saved. */ @Override public void Save() { diff --git a/testar/src/org/testar/extendedsettings/ExtendedSettingContainer.java b/testar/src/org/testar/extendedsettings/ExtendedSettingContainer.java index d14386bdb..954ed3bea 100644 --- a/testar/src/org/testar/extendedsettings/ExtendedSettingContainer.java +++ b/testar/src/org/testar/extendedsettings/ExtendedSettingContainer.java @@ -48,6 +48,7 @@ public ExtendedSettingContainer(ExtendedSettingFile file, Class clazz, /** * Get the settings. + * * @return The actual settings. */ ExtendedSettingBase GetSettings() { diff --git a/testar/src/org/testar/extendedsettings/ExtendedSettingFile.java b/testar/src/org/testar/extendedsettings/ExtendedSettingFile.java index 860b7b9f2..3065dba56 100644 --- a/testar/src/org/testar/extendedsettings/ExtendedSettingFile.java +++ b/testar/src/org/testar/extendedsettings/ExtendedSettingFile.java @@ -32,9 +32,9 @@ import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang.SerializationUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Level; import org.checkerframework.checker.nullness.qual.NonNull; +import org.testar.Logger; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -90,8 +90,8 @@ public ExtractionResult(JAXBContext context, RootSetting data, Boolean fileNotFo } public class ExtendedSettingFile implements Serializable { - private static final Logger LOGGER = LogManager.getLogger(); public static final String FileName = "ExtendedSettings.xml"; + private static final String TAG = "ExtendedSettings"; private final String _absolutePath; private final ReentrantReadWriteLock _fileAccessMutex; @@ -105,10 +105,10 @@ public class ExtendedSettingFile implements Serializable { /** * Constructor, each specialization must have a unique implementation of this class. * - * @param fileLocation The absolute path the the XML file. + * @param fileLocation The absolute path of the XML file. * @param fileAccessMutex Mutex for thread-safe access. */ - public ExtendedSettingFile(@NonNull String fileLocation, @NonNull ReentrantReadWriteLock fileAccessMutex) { + public ExtendedSettingFile(@NonNull String fileLocation, @NonNull ReentrantReadWriteLock fileAccessMutex) { _fileAccessMutex = fileAccessMutex; _absolutePath = System.getProperty("user.dir") + (fileLocation.startsWith(".") ? fileLocation.substring(1) : (fileLocation.startsWith(File.separator) @@ -137,14 +137,14 @@ public T load(@NonNull Class clazz) { // We only support loading a single element for now. if (rd.Data.any.stream().filter(element -> element.getClass() == clazz).count() > 1) { - LOGGER.error("Duplicate elements found for {}, returning first element ", clazz); + Logger.log(Level.ERROR, TAG, "Duplicate elements found for {}, returning first element ", clazz); } // Store the current content, so we can replace it when needed. _loadedValue = SerializationUtils.clone((Serializable) result); } if (result == null) { - LOGGER.info("Did not found XML element for class: {}", clazz); + Logger.log(Level.INFO, TAG, "Did not found XML element for class: {}", clazz); } return result; @@ -163,7 +163,7 @@ public T load(@NonNull Class clazz, @NonNull IExtendedSettingDefaultValue T result = load(clazz); if (result == null) { - LOGGER.info("Writing default values for {}", clazz); + Logger.log(Level.TRACE, TAG, "Writing default values for {}", clazz); save(defaultFunctor.CreateDefault()); return load(clazz); } @@ -179,7 +179,7 @@ public T load(@NonNull Class clazz, @NonNull IExtendedSettingDefaultValue */ public void save(@NonNull Object data) { if (!(data instanceof Comparable)) { - LOGGER.error("Object {} is not extending Comparable", data); + Logger.log(Level.ERROR, TAG, "Object {} is not extending Comparable", data); return; } @@ -281,7 +281,7 @@ private void createExtendedSettingsFile() { Marshaller marshaller = context.createMarshaller(); marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); marshaller.marshal(new RootSetting(), os); - LOGGER.info("Created extended settings file: {}", _absolutePath); + Logger.log(Level.INFO, TAG, "Created extended settings file: {}", _absolutePath); } catch (JAXBException e) { e.printStackTrace(); } finally { diff --git a/testar/src/org/testar/extendedsettings/ExtendedSettingsFactory.java b/testar/src/org/testar/extendedsettings/ExtendedSettingsFactory.java index 6e03990b4..d8fc0ca8f 100644 --- a/testar/src/org/testar/extendedsettings/ExtendedSettingsFactory.java +++ b/testar/src/org/testar/extendedsettings/ExtendedSettingsFactory.java @@ -31,6 +31,8 @@ package org.testar.extendedsettings; import org.testar.visualvalidation.VisualValidationSettings; +import org.testar.visualvalidation.extractor.WidgetTextConfiguration; +import org.testar.visualvalidation.ocr.tesseract.TesseractSettings; import java.util.ArrayList; import java.util.List; @@ -84,4 +86,12 @@ public static VisualValidationSettings createVisualValidationSettings() { public static ExampleSetting createTestSetting() { return createSettings(ExampleSetting.class, ExampleSetting::CreateDefault); } -} \ No newline at end of file + + public static TesseractSettings createTesseractSetting() { + return createSettings(TesseractSettings.class, TesseractSettings::CreateDefault); + } + + public static WidgetTextConfiguration createWidgetTextConfiguration() { + return createSettings(WidgetTextConfiguration.class, WidgetTextConfiguration::CreateDefault); + } +} diff --git a/testar/src/org/testar/monkey/ConfigTags.java b/testar/src/org/testar/monkey/ConfigTags.java index 5eec9d38b..a587ac742 100644 --- a/testar/src/org/testar/monkey/ConfigTags.java +++ b/testar/src/org/testar/monkey/ConfigTags.java @@ -33,6 +33,7 @@ import org.testar.monkey.alayer.Tag; +import java.nio.file.Path; import java.util.List; public final class ConfigTags { diff --git a/testar/src/org/testar/monkey/DefaultProtocol.java b/testar/src/org/testar/monkey/DefaultProtocol.java index 4966ab9d2..7d508f44d 100644 --- a/testar/src/org/testar/monkey/DefaultProtocol.java +++ b/testar/src/org/testar/monkey/DefaultProtocol.java @@ -31,41 +31,45 @@ package org.testar.monkey; -import static org.testar.monkey.alayer.Tags.ActionDelay; -import static org.testar.monkey.alayer.Tags.ActionDuration; -import static org.testar.monkey.alayer.Tags.ActionSet; -import static org.testar.monkey.alayer.Tags.Desc; -import static org.testar.monkey.alayer.Tags.ExecutedAction; -import static org.testar.monkey.alayer.Tags.IsRunning; -import static org.testar.monkey.alayer.Tags.OracleVerdict; -import static org.testar.monkey.alayer.Tags.SystemState; - -import java.awt.Desktop; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.PrintStream; -import java.util.*; -import java.util.logging.Level; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; - -import javax.swing.JFrame; -import javax.swing.JOptionPane; -import org.testar.*; -import org.testar.reporting.Reporting; -import org.testar.statemodel.StateModelManager; -import org.testar.statemodel.StateModelManagerFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.testar.monkey.alayer.*; +import org.jnativehook.GlobalScreen; +import org.jnativehook.NativeHookException; +import org.openqa.selenium.SessionNotCreatedException; +import org.testar.ActionStatus; +import org.testar.CodingManager; +import org.testar.EventHandler; +import org.testar.FileHandling; +import org.testar.OutputStructure; +import org.testar.ProcessInfo; +import org.testar.ProcessListener; +import org.testar.ProtocolUtil; +import org.testar.RandomActionSelector; +import org.testar.SutVisualization; +import org.testar.SystemProcessHandling; +import org.testar.extendedsettings.ExtendedSettingsFactory; +import org.testar.managers.DataManager; +import org.testar.monkey.alayer.AWTCanvas; +import org.testar.monkey.alayer.Action; +import org.testar.monkey.alayer.AutomationCache; +import org.testar.monkey.alayer.Canvas; +import org.testar.monkey.alayer.Color; +import org.testar.monkey.alayer.FillPattern; +import org.testar.monkey.alayer.Finder; +import org.testar.monkey.alayer.Pen; +import org.testar.monkey.alayer.Roles; +import org.testar.monkey.alayer.SUT; +import org.testar.monkey.alayer.Shape; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.StateBuilder; +import org.testar.monkey.alayer.StrokePattern; +import org.testar.monkey.alayer.Tag; +import org.testar.monkey.alayer.Taggable; +import org.testar.monkey.alayer.TaggableBase; +import org.testar.monkey.alayer.Tags; +import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.Visualizer; +import org.testar.monkey.alayer.Widget; import org.testar.monkey.alayer.actions.ActivateSystem; import org.testar.monkey.alayer.actions.AnnotatingActionCompiler; import org.testar.monkey.alayer.actions.KillProcess; @@ -84,13 +88,56 @@ import org.testar.monkey.alayer.windows.WinApiException; import org.testar.plugin.NativeLinker; import org.testar.plugin.OperatingSystems; -import org.testar.managers.DataManager; +import org.testar.reporting.Reporting; import org.testar.serialisation.LogSerialiser; import org.testar.serialisation.ScreenshotSerialiser; import org.testar.serialisation.TestSerialiser; -import org.jnativehook.GlobalScreen; -import org.jnativehook.NativeHookException; -import org.openqa.selenium.SessionNotCreatedException; +import org.testar.statemodel.StateModelManager; +import org.testar.statemodel.StateModelManagerFactory; +import org.testar.visualvalidation.VisualValidationFactory; +import org.testar.visualvalidation.VisualValidationManager; +import org.testar.visualvalidation.VisualValidationTag; + +import javax.swing.JEditorPane; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JOptionPane; +import javax.swing.event.HyperlinkEvent; +import java.awt.Desktop; +import java.awt.Font; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.WeakHashMap; +import java.util.logging.Level; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; + +import static org.testar.monkey.alayer.Tags.ActionDelay; +import static org.testar.monkey.alayer.Tags.ActionDuration; +import static org.testar.monkey.alayer.Tags.ActionSet; +import static org.testar.monkey.alayer.Tags.Desc; +import static org.testar.monkey.alayer.Tags.ExecutedAction; +import static org.testar.monkey.alayer.Tags.IsRunning; +import static org.testar.monkey.alayer.Tags.OracleVerdict; +import static org.testar.monkey.alayer.Tags.OriginWidget; +import static org.testar.monkey.alayer.Tags.SystemState; public class DefaultProtocol extends RuntimeControlsProtocol { @@ -152,7 +199,6 @@ protected final double timeElapsed() { protected List contextRunningProcesses = null; protected static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; - protected static final Logger INDEXLOG = LogManager.getLogger(); protected double passSeverity = Verdict.SEVERITY_OK; public static Action lastExecutedAction = null; @@ -173,10 +219,11 @@ protected final double timeElapsed() { protected StateModelManager stateModelManager; private String startOfSutDateString; //value set when SUT started, used for calculating the duration of test + protected VisualValidationManager visualValidationManager; + // Creating a logger with log4j library: private static Logger logger = LogManager.getLogger(); - /** * This is the abstract flow of TESTAR (generate mode): * @@ -265,15 +312,15 @@ public final void run(final Settings settings) { if(e.getMessage()!=null && e.getMessage().contains("Chrome version")) { String msg = "*** Unsupported versions exception: Chrome browser and Selenium WebDriver versions *** \n" - + "Please verify your Chrome browser version: chrome://settings/help \n" - + "And download the appropriate ChromeDriver version: https://chromedriver.chromium.org/downloads \n" - + "\n" - //TODO check when implementing other webdriver than chromedriver + + "
Please verify your Chrome browser version: chrome://settings/help" + + "
And download the appropriate ChromeDriver version." + + "
" + //TODO check when implementing other webdriver than chromedriver //TODO remove when automatically killing webdriver process when creating the session fails - + "As a result of this error, there is probably a \"chromedriver.exe\" process running. \n" - + "Please use Windows Task Manager to stop that process."; + + "
As a result of this error, there is probably a \"chromedriver.exe\" process running." + + "
Please use Windows Task Manager to stop that process."; - popupMessage(msg); + chromeDriverMissing(msg); System.out.println(msg); System.out.println(e.getMessage()); }else { @@ -335,6 +382,8 @@ protected void initialize(Settings settings) { // new state model manager stateModelManager = StateModelManagerFactory.getStateModelManager(settings); + + visualValidationManager = VisualValidationFactory.createVisualValidator(settings.get(ConfigTags.ProtocolClass)); } try { @@ -440,6 +489,47 @@ private void popupMessage(String message) { if(settings.get(ConfigTags.ShowVisualSettingsDialogOnStartup)) { JFrame frame = new JFrame(); JOptionPane.showMessageDialog(frame, message); + + } + } + + /** + * Show a popup containing a html message with click interaction. + * Only if GUI option is enabled (disabled for CI) + */ + private void chromeDriverMissing(String htmlMessage) { + if(settings.get(ConfigTags.ShowVisualSettingsDialogOnStartup)) { + JFrame frame = new JFrame(); + // for copying style + JLabel label = new JLabel(); + Font font = label.getFont(); + + // create some css from the label's font + String style = "font-family:" + font.getFamily() + ";" + + "font-weight:" + (font.isBold() ? "bold" : "normal") + ";" + + "font-size:" + font.getSize() + "pt;"; + // html content + JEditorPane ep = new JEditorPane("text/html", "" + + htmlMessage + + ""); + + // handle link events + ep.addHyperlinkListener(e -> { + if (e.getEventType().equals(HyperlinkEvent.EventType.ACTIVATED)) { + Desktop desktop = Desktop.getDesktop(); + try { + desktop.browse(e.getURL().toURI()); + } catch (URISyntaxException | IOException exception) { + exception.printStackTrace(); + } + } + }); + ep.setEditable(false); + ep.setBackground(label.getBackground()); + + // show + JOptionPane.showMessageDialog(frame, ep); + } } @@ -909,17 +999,16 @@ protected void runSpyLoop() { Set actions = deriveActions(system,state); buildStateActionsIdentifiers(state, actions); - //in Spy-mode, always visualize the widget info under the mouse cursor: SutVisualization.visualizeState(visualizationOn, markParentWidget, mouse, lastPrintParentsOf, cv, state); //in Spy-mode, always visualize the green dots: visualizeActions(cv, state, actions); - + cv.end(); int msRefresh = (int)(settings.get(ConfigTags.RefreshSpyCanvas, 0.5) * 1000); - + synchronized (this) { try { this.wait(msRefresh); @@ -937,7 +1026,7 @@ protected void runSpyLoop() { Util.clear(cv); cv.end(); - + //finishSequence() content, but SPY mode is not a sequence if(!NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { SystemProcessHandling.killTestLaunchedProcesses(this.contextRunningProcesses); @@ -1142,7 +1231,9 @@ protected void runReplayLoop(){ double rrt = settings.get(ConfigTags.ReplayRetryTime); - while(success && !faultySequence && mode() == Modes.Replay){ + boolean suppressFaultySequence = ExtendedSettingsFactory.createVisualValidationSettings().enabled; + + while(success && (suppressFaultySequence || !faultySequence) && mode() == Modes.Replay){ //Initialize local fragment and read saved action of PathToReplaySequence File Taggable replayableFragment; @@ -1436,14 +1527,15 @@ protected State getState(SUT system) throws StateBuildException { setStateForClickFilterLayerProtocol(state); - if(settings.get(ConfigTags.Mode) == Modes.Spy) + if(settings.get(ConfigTags.Mode) == Modes.Spy) { return state; - - Verdict verdict = getVerdict(state); - state.set(Tags.OracleVerdict, verdict); + } setStateScreenshot(state); + Verdict verdict = getVerdict(state); + state.set(Tags.OracleVerdict, verdict); + if (mode() != Modes.Spy && verdict.severity() >= settings().get(ConfigTags.FaultThreshold)){ faultySequence = true; LogSerialiser.log("Detected fault: " + verdict + "\n", LogSerialiser.LogLevel.Critical); @@ -1457,29 +1549,37 @@ protected State getState(SUT system) throws StateBuildException { passSeverity = verdict.severity(); LogSerialiser.log("Detected warning: " + verdict + "\n", LogSerialiser.LogLevel.Critical); } - + return state; } /** - * Take a Screenshot of the State and associate the path into state tag + * Take a Screenshot of the State and associate the path into state tag. + * If enabled run the visual validation on the capture screenshot. */ private void setStateScreenshot(State state) { Shape viewPort = state.get(Tags.Shape, null); + AWTCanvas screenshot = null; if(viewPort != null){ if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - //System.out.println("DEBUG: Using WebDriver specific state shot."); - state.set(Tags.ScreenshotPath, WdProtocolUtil.getStateshot(state)); + screenshot = WdProtocolUtil.getStateshotBinary(state); }else{ - //System.out.println("DEBUG: normal state shot"); - state.set(Tags.ScreenshotPath, ProtocolUtil.getStateshot(state)); + screenshot = ProtocolUtil.getStateshotBinary(state); } - } - } + String screenshotPath = ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom, + "NoConcreteIdAvailable"), screenshot); + state.set(Tags.ScreenshotPath, screenshotPath); + } - @Override + htmlReport.addVisualValidationResult( + visualValidationManager.AnalyzeImage(state, screenshot), state, null + ); + } + + @Override protected Verdict getVerdict(State state){ Assert.notNull(state); + Verdict visualValidationVerdict = state.get(VisualValidationTag.VisualValidationVerdict, Verdict.OK); //------------------- // ORACLES FOR FREE //------------------- @@ -1500,18 +1600,18 @@ protected Verdict getVerdict(State state){ this.suspiciousTitlesPattern = Pattern.compile(settings().get(ConfigTags.SuspiciousTitles), Pattern.UNICODE_CHARACTER_CLASS); // search all widgets for suspicious String Values - Verdict suspiciousValueVerdict = Verdict.OK; + Verdict suspiciousValueVerdict; for(Widget w : state) { suspiciousValueVerdict = suspiciousStringValueMatcher(w); if(suspiciousValueVerdict.severity() == Verdict.SEVERITY_SUSPICIOUS_TITLE) { - return suspiciousValueVerdict; + return suspiciousValueVerdict.join(visualValidationVerdict); } } // if everything was OK ... - return Verdict.OK; + return Verdict.OK.join(visualValidationVerdict); } - + private Verdict suspiciousStringValueMatcher(Widget w) { Matcher m; @@ -1635,15 +1735,8 @@ protected Set preSelectAction(SUT system, State state, Set actio //TODO move the CPU metric to another helper class that is not default "TrashBinCode" or "SUTprofiler" //TODO check how well the CPU usage based waiting works protected boolean executeAction(SUT system, State state, Action action){ + takeActionScreenshot(state, action); - if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)){ - //System.out.println("DEBUG: Using WebDriver specific action shot."); - WdProtocolUtil.getActionshot(state,action); - }else{ - //System.out.println("DEBUG: normal action shot"); - ProtocolUtil.getActionshot(state,action); - } - double waitTime = settings.get(ConfigTags.TimeToWaitAfterAction); try{ @@ -1668,16 +1761,11 @@ protected boolean executeAction(SUT system, State state, Action action){ return false; } } - + protected boolean replayAction(SUT system, State state, Action action, double actionWaitTime, double actionDuration){ - // Get an action screenshot based on the NativeLinker platform - if(NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { - WdProtocolUtil.getActionshot(state,action); - } else { - ProtocolUtil.getActionshot(state,action); - } + takeActionScreenshot(state, action); - try{ + try{ double halfWait = actionWaitTime == 0 ? 0.01 : actionWaitTime / 2.0; // seconds Util.pause(halfWait); // help for a better match of the state' actions visualization action.run(system, state, actionDuration); @@ -1700,6 +1788,24 @@ protected boolean replayAction(SUT system, State state, Action action, double ac } } + private void takeActionScreenshot(State state, Action action) { + AWTCanvas screenshot; + if (NativeLinker.getPLATFORM_OS().contains(OperatingSystems.WEBDRIVER)) { + screenshot = WdProtocolUtil.getActionshot(state, action); + } else { + screenshot = ProtocolUtil.getActionshot(state, action); + } + state.set(ExecutedAction, action); + ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), + screenshot); + + htmlReport.addVisualValidationResult( + visualValidationManager.AnalyzeImage(state, screenshot, action.get(OriginWidget, null)), + state, action + ); + } + /** * This method is here, so that ClickFilterLayerProtocol can override it, and the behaviour is updated * @@ -1734,7 +1840,10 @@ protected String getRandomText(Widget w){ * @return */ protected boolean moreActions(State state) { - return (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence) && + // When visual validation module is enabled continue even when faults are detected. + boolean suppressFaultySequence = ExtendedSettingsFactory.createVisualValidationSettings().enabled || + (!settings().get(ConfigTags.StopGenerationOnFault) || !faultySequence); + return suppressFaultySequence && state.get(Tags.IsRunning, false) && !state.get(Tags.NotResponding, false) && //actionCount() < settings().get(ConfigTags.SequenceLength) && actionCount() <= lastSequenceActionNumber && @@ -1786,6 +1895,9 @@ private void closeTestarTestSession(){ GlobalScreen.unregisterNativeHook(); } } + if (visualValidationManager != null) { + visualValidationManager.Destroy(); + } } catch (NativeHookException e) { e.printStackTrace(); } catch (NullPointerException e) { diff --git a/testar/src/org/testar/protocols/DesktopProtocol.java b/testar/src/org/testar/protocols/DesktopProtocol.java index 84607eb5d..496c8f7a3 100644 --- a/testar/src/org/testar/protocols/DesktopProtocol.java +++ b/testar/src/org/testar/protocols/DesktopProtocol.java @@ -31,6 +31,7 @@ package org.testar.protocols; +import org.apache.logging.log4j.Level; import org.testar.DerivedActions; import org.testar.reporting.Reporting; import org.testar.monkey.Drag; @@ -42,18 +43,20 @@ import org.testar.monkey.alayer.exceptions.StateBuildException; import org.testar.monkey.ConfigTags; import org.testar.OutputStructure; +import org.testar.Logger; import java.io.File; import java.io.IOException; import java.util.HashSet; import java.util.Set; + import static org.testar.monkey.alayer.Tags.Blocked; import static org.testar.monkey.alayer.Tags.Enabled; public class DesktopProtocol extends GenericUtilsProtocol { + private static final String TAG = "DesktopProtocol"; //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness - protected Reporting htmlReport; protected State latestState; /** @@ -186,7 +189,7 @@ protected void postSequenceProcessing() { statusInfo = statusInfo.replace("\n"+ Verdict.OK.info(), ""); //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" - INDEXLOG.info(OutputStructure.executedSUTname + Logger.log(Level.INFO, TAG, OutputStructure.executedSUTname + " " + settings.get(ConfigTags.Mode, mode()) + " " + sequencesPath + " " + status + " \"" + statusInfo + "\"" ); diff --git a/testar/src/org/testar/protocols/WebdriverProtocol.java b/testar/src/org/testar/protocols/WebdriverProtocol.java index c3b95055a..0c58e7777 100644 --- a/testar/src/org/testar/protocols/WebdriverProtocol.java +++ b/testar/src/org/testar/protocols/WebdriverProtocol.java @@ -49,11 +49,11 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.Level; import org.openqa.selenium.logging.LogEntries; import org.openqa.selenium.logging.LogEntry; import org.openqa.selenium.logging.LogType; @@ -78,12 +78,13 @@ import org.testar.OutputStructure; import org.testar.serialisation.LogSerialiser; import org.testar.reporting.Reporting; +import org.testar.Logger; public class WebdriverProtocol extends GenericUtilsProtocol { - //Attributes for adding slide actions + private static final String TAG = "WebdriverProtocol"; + //Attributes for adding slide actions protected static double SCROLL_ARROW_SIZE = 36; // sliding arrows protected static double SCROLL_THICK = 16; //scroll thickness - protected Reporting htmlReport; protected State latestState; protected static Set existingCssClasses = new HashSet<>(); @@ -329,7 +330,7 @@ protected Verdict getVerdict(State state) { RemoteWebDriver driver = WdDriver.getRemoteWebDriver(); LogEntries logEntries = driver.manage().logs().get(LogType.BROWSER); for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.SEVERE)) { + if(logEntry.getLevel().equals(java.util.logging.Level.SEVERE)) { // Check if the severe error message matches with the web console error pattern String consoleErrorMsg = logEntry.getMessage(); Matcher matcherError = errorPattern.matcher(consoleErrorMsg); @@ -350,7 +351,7 @@ protected Verdict getVerdict(State state) { RemoteWebDriver driver = WdDriver.getRemoteWebDriver(); LogEntries logEntries = driver.manage().logs().get(LogType.BROWSER); for(LogEntry logEntry : logEntries) { - if(logEntry.getLevel().equals(Level.WARNING)) { + if(logEntry.getLevel().equals(java.util.logging.Level.WARNING)) { // Check if the warning message matches with the web console error pattern String consoleWarningMsg = logEntry.getMessage(); Matcher matcherWarning = warningPattern.matcher(consoleWarningMsg); @@ -456,13 +457,13 @@ protected void postSequenceProcessing() { statusInfo = statusInfo.replace("\n"+Verdict.OK.info(), ""); - //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" - INDEXLOG.info(OutputStructure.executedSUTname - + " " + settings.get(ConfigTags.Mode, mode()) - + " " + sequencesPath - + " " + status + " \"" + statusInfo + "\"" ); - - htmlReport.close(); + //Timestamp(generated by logger) SUTname Mode SequenceFileObject Status "StatusInfo" + Logger.log(Level.INFO, TAG, OutputStructure.executedSUTname + + " " + settings.get(ConfigTags.Mode, mode()) + + " " + sequencesPath + + " " + status + " \"" + statusInfo + "\"" ); + + htmlReport.close(); } @Override diff --git a/testar/src/org/testar/reporting/HtmlSequenceReport.java b/testar/src/org/testar/reporting/HtmlSequenceReport.java index 3b1e2e8f6..82f047cc5 100644 --- a/testar/src/org/testar/reporting/HtmlSequenceReport.java +++ b/testar/src/org/testar/reporting/HtmlSequenceReport.java @@ -32,10 +32,16 @@ package org.testar.reporting; import org.apache.commons.lang.StringEscapeUtils; +import org.checkerframework.checker.nullness.qual.Nullable; import org.testar.monkey.alayer.Action; import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Tags; import org.testar.monkey.alayer.Verdict; +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.matcher.CharacterMatchEntry; +import org.testar.visualvalidation.matcher.ContentMatchResult; +import org.testar.visualvalidation.matcher.LocationMatch; +import org.testar.visualvalidation.matcher.MatcherResult; import org.testar.OutputStructure; import org.testar.monkey.alayer.exceptions.NoSuchTagException; @@ -43,9 +49,11 @@ import java.io.FileNotFoundException; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; +import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; -public class HtmlSequenceReport implements Reporting{ +public class HtmlSequenceReport implements Reporting { private boolean firstStateAdded = false; private boolean firstActionsAdded = false; @@ -60,11 +68,24 @@ public class HtmlSequenceReport implements Reporting{ "if(direction === 'column') document.getElementById('main').style.flexDirection = 'column-reverse';", "else document.getElementById('main').style.flexDirection = 'column';}", "", + "", "", "TESTAR execution sequence report", "", "" }; + private static final String DIV_ID_BLOCK_START = "
"; + private static final String DIV_CLOSE = "
"; private PrintWriter out; private static final String REPORT_FILENAME_MID = "_sequence_"; @@ -75,15 +96,15 @@ public class HtmlSequenceReport implements Reporting{ private int innerLoopCounter = 0; public HtmlSequenceReport() { - try{ + try { //TODO put filename into settings, name with sequence number // creating a new file for the report - htmlFilename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString+"_" + htmlFilename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname + REPORT_FILENAME_MID + OutputStructure.sequenceInnerLoopCount + REPORT_FILENAME_AFT; out = new PrintWriter(htmlFilename, HTMLReporter.CHARSET); - for(String s:HEADER){ + for (String s : HEADER) { write(s); } @@ -107,12 +128,12 @@ public HtmlSequenceReport() { */ public HtmlSequenceReport(String pathReplayedSequence) { try { - htmlFilename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString+"_" + htmlFilename = OutputStructure.htmlOutputDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname + REPORT_FILENAME_MID + OutputStructure.sequenceInnerLoopCount + REPORT_FILENAME_AFT; out = new PrintWriter(htmlFilename, HTMLReporter.CHARSET); - for(String s:HEADER) { + for (String s : HEADER) { write(s); } @@ -125,213 +146,323 @@ public HtmlSequenceReport(String pathReplayedSequence) { } } - public void addTitle(int h, String text){ - write(""+text+""); + private static String correctScreenshotPath(String screenshotDir) { + if (screenshotDir.contains("./output")) { + int indexStart = screenshotDir.indexOf("./output"); + int indexScrn = screenshotDir.indexOf("scrshots"); + String replaceString = screenshotDir.substring(indexStart, indexScrn); + screenshotDir = screenshotDir.replace(replaceString, "../"); + } + return screenshotDir; + } + + static private String getScreenshotPath(State state) { + return correctScreenshotPath(state.get(Tags.ScreenshotPath)); + } + + static private String getActionScreenshotPath(State state, Action action) { + final String screenshotDir = correctScreenshotPath(OutputStructure.screenshotsOutputDir); + String actionPath = screenshotDir + File.separator + OutputStructure.startInnerLoopDateString + "_" + + OutputStructure.executedSUTname + "_sequence_" + OutputStructure.sequenceInnerLoopCount + + File.separator + state.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + "_" + + action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + ".png"; + + if (actionPath.contains("./output")) { + actionPath = actionPath.replace("./output", ".."); + } + return actionPath; + } + + public void addTitle(int h, String text) { + write("" + text + ""); } //TODO: This method is not used, check and delete - public void addSequenceStep(State state, String actionImagePath){ - try { - String imagePath = state.get(Tags.ScreenshotPath); - // repairing the file paths: - if(imagePath.contains("./output")){ - imagePath = imagePath.replace("./output","../"); - } - write("

State:

"); - write("

"); - write("

Action:

"); - write("

"); - }catch(NullPointerException | NoSuchTagException e) { - System.out.println("ERROR: Adding the Sequence step " + innerLoopCounter + " in the HTML report"); - write("

ERROR Adding current Sequence step " + innerLoopCounter + "

"); - } + public void addSequenceStep(State state, String actionImagePath) { + try { + String imagePath = state.get(Tags.ScreenshotPath); + // repairing the file paths: + if (imagePath.contains("./output")) { + imagePath = imagePath.replace("./output", "../"); + } + write("

State:

"); + write("

"); + write("

Action:

"); + write("

"); + } catch (NullPointerException | NoSuchTagException e) { + System.out.println("ERROR: Adding the Sequence step " + innerLoopCounter + " in the HTML report"); + write("

ERROR Adding current Sequence step " + innerLoopCounter + "

"); + } } - public void addState(State state){ - if(firstStateAdded){ - if(firstActionsAdded){ + public void addState(State state) { + if (firstStateAdded) { + if (firstActionsAdded) { writeStateIntoReport(state); - }else{ + } else { //don't write the state as it is the same - getState is run twice in the beginning, before the first action } - }else{ + } else { firstStateAdded = true; writeStateIntoReport(state); } } - private void writeStateIntoReport(State state){ - try { - String imagePath = state.get(Tags.ScreenshotPath); - if(imagePath.contains("./output")){ - int indexStart = imagePath.indexOf("./output"); - int indexScrn = imagePath.indexOf("scrshots"); - String replaceString = imagePath.substring(indexStart,indexScrn); - imagePath = imagePath.replace(replaceString,"../"); - } - write("
"); // Open state block container - write("

State "+innerLoopCounter+"

"); - write("

ConcreteIDCustom="+state.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable")+"

"); - write("

AbstractIDCustom="+state.get(Tags.AbstractIDCustom, "NoAbstractIdCustomAvailable")+"

"); - write("

"); - write("
"); // Close state block container - }catch(NullPointerException | NoSuchTagException e) { - System.out.println("ERROR: Adding the State number " + innerLoopCounter + " in the HTML report"); - write("

ERROR Adding current State " + innerLoopCounter + "

"); - } - innerLoopCounter++; + private void writeStateIntoReport(State state) { + try { + write(DIV_ID_BLOCK_START); // Open state block container + write("

State " + innerLoopCounter + "

"); + write("

ConcreteIDCustom=" + state.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + "

"); + write("

AbstractIDCustom=" + state.get(Tags.AbstractIDCustom, "NoAbstractIdCustomAvailable") + "

"); + write("

"); //Smiley face + write(DIV_CLOSE); // Close state block container + } catch (NullPointerException | NoSuchTagException e) { + System.out.println("ERROR: Adding the State number " + innerLoopCounter + " in the HTML report"); + write("

ERROR Adding current State " + innerLoopCounter + "

"); + } + innerLoopCounter++; } - public void addActions(Set actions){ - if(!firstActionsAdded) firstActionsAdded = true; - write("
"); // Open derived actions block container + public void addActions(Set actions) { + if (!firstActionsAdded) firstActionsAdded = true; + write(DIV_ID_BLOCK_START); // Open derived actions block container write("

Set of actions:

    "); - for(Action action:actions){ + for (Action action : actions) { write("
  • "); - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write(""+ escaped +" || "); - } - }catch(NullPointerException | NoSuchTagException e){ + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + " || "); + } + } catch (NullPointerException | NoSuchTagException e) { e.printStackTrace(); } write(StringEscapeUtils.escapeHtml(action.toString())); - write(" || ConcreteIDCustom="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable")); - try{if(action.get(Tags.AbstractIDCustom)!=null) write(" || AbstractIDCustom="+action.get(Tags.AbstractIDCustom));}catch(NullPointerException | NoSuchTagException e){e.printStackTrace();} + write(" || ConcreteId=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable")); + try { + if (action.get(Tags.AbstractIDCustom) != null) write(" || AbstractIdCustom=" + action.get(Tags.AbstractIDCustom)); + } catch(NullPointerException | NoSuchTagException e) { + e.printStackTrace(); + } + write("
  • "); } write("
"); - write("
"); // Close derived actions block container + write(DIV_CLOSE); // Close derived actions block container } - public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions){ - if(!firstActionsAdded) firstActionsAdded = true; - if(actions.size()==concreteIdsOfUnvisitedActions.size()){ - write("
"); // Open derived actions block container + public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions) { + if (!firstActionsAdded) firstActionsAdded = true; + if (actions.size() == concreteIdsOfUnvisitedActions.size()) { + write(DIV_ID_BLOCK_START); // Open derived actions block container write("

Set of actions (all unvisited - a new state):

    "); - for(Action action:actions){ + for (Action action : actions) { write("
  • "); - - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(NullPointerException | NoSuchTagException e){} - - write(" || ConcreteIDCustom="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); - + + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (NullPointerException | NoSuchTagException e) { + } + + write(" || ConcreteIDCustom=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); + write("
  • "); } write("
"); - write("
"); // Close derived actions block container - }else if(concreteIdsOfUnvisitedActions.size()==0){ - write("
"); // Open derived actions block container + write(DIV_CLOSE); // Close derived actions block container + } else if (concreteIdsOfUnvisitedActions.size() == 0) { + write(DIV_ID_BLOCK_START); // Open derived actions block container write("

All actions have been visited, set of available actions:

    "); - for(Action action:actions){ - write("
  • "); + for (Action action : actions) { + write("
  • "); - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(NullPointerException | NoSuchTagException e){} + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (NullPointerException | NoSuchTagException e) { + } - write(" || ConcreteIDCustom="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); + write(" || ConcreteIDCustom=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); - write("
  • "); + write(""); } write("
"); - write("
"); // Close derived actions block container - }else{ - write("
"); // Open derived actions block container - write("

"+concreteIdsOfUnvisitedActions.size()+" out of "+actions.size()+" actions have not been visited yet:

    "); - for(Action action:actions){ - if(concreteIdsOfUnvisitedActions.contains(action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable"))){ - //action is unvisited -> showing: - write("
  • "); - - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write("" + escaped + ""); - } - }catch(NullPointerException | NoSuchTagException e){} - - write(" || ConcreteIDCustom="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") - + " || " + StringEscapeUtils.escapeHtml(action.toString())); - - write("
  • "); - } + write(DIV_CLOSE); // Close derived actions block container + } else { + write(DIV_ID_BLOCK_START); // Open derived actions block container + write("

    " + concreteIdsOfUnvisitedActions.size() + " out of " + actions.size() + " actions have not been visited yet:

      "); + for (Action action : actions) { + if (concreteIdsOfUnvisitedActions.contains(action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"))) { + //action is unvisited -> showing: + write("
    • "); + + try { + if (action.get(Tags.Desc) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write("" + escaped + ""); + } + } catch (NullPointerException | NoSuchTagException e) { + } + + write(" || ConcreteIDCustom=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable") + + " || " + StringEscapeUtils.escapeHtml(action.toString())); + + write("
    • "); + } } write("
    "); - write("
"); // Close derived actions block container + write(DIV_CLOSE); // Close derived actions block container } } - public void addSelectedAction(State state, Action action){ - String screenshotDir = OutputStructure.screenshotsOutputDir; + public void addSelectedAction(State state, Action action) { + write(DIV_ID_BLOCK_START); // Open executed action block container + write("

Selected Action " + innerLoopCounter + " leading to State " + innerLoopCounter + "\"

"); + write("

ConcreteIDCustom=" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable")); - if(screenshotDir.contains("./output")){ - int indexStart = screenshotDir.indexOf("./output"); - int indexScrn = screenshotDir.indexOf("scrshots"); - String replaceString = screenshotDir.substring(indexStart,indexScrn); - screenshotDir = screenshotDir.replace(replaceString,"../"); + if (action.get(Tags.Desc, null) != null) { + String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); + write(" || " + escaped); } - String actionPath = screenshotDir + File.separator - + OutputStructure.startInnerLoopDateString + "_" + OutputStructure.executedSUTname - + "_sequence_" + OutputStructure.sequenceInnerLoopCount - + File.separator + state.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + "_" + action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable") + ".png"; + write("

"); + write("

"); //Smiley face + write(DIV_CLOSE); // Close executed action block container + } + + public void addTestVerdict(Verdict verdict) { + String verdictInfo = verdict.info(); + if (verdict.severity() > Verdict.OK.severity()) + verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); + + write(DIV_ID_BLOCK_START); // Open verdict block container + write("

Test verdict for this sequence: " + verdictInfo + "

"); + write("

Severity: " + verdict.severity() + "

"); + write(DIV_CLOSE); // Close verdict block container + + FINAL_VERDICT_FILENAME = verdict.verdictSeverityTitle(); + } - write("
"); // Open executed action block container - write("

Selected Action "+innerLoopCounter+" leading to State "+innerLoopCounter+"\"

"); - write("

ConcreteIDCustom="+action.get(Tags.ConcreteIDCustom, "NoConcreteIdCustomAvailable")); + @Override + public void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action) { + if (result != null) { + write(DIV_ID_BLOCK_START); + if (!result.getNoLocationMatches().isEmpty() || !result.getResult().isEmpty()) { + write("

Visual validation result:

"); + + // Add the annotated screenshot + StringBuilder screenshotPath = new StringBuilder(action != null ? + getActionScreenshotPath(state, action) : getScreenshotPath(state)); + screenshotPath.insert(screenshotPath.indexOf(".png"), MatcherResult.ScreenshotPostFix); + write("

"); + + // List the expected elements without a location match. + result.getNoLocationMatches().forEach(it -> { + String type = it instanceof ExpectedElement ? "expected" : "ocr"; + write("

No match for " + type + ": \"" + it._text + "\" at location: " + it._location + "

"); + } + ); + + // List the expected elements with a location match. + composeMatchedResultTable(result); + } + } else { + write("

Visual validation result:

"); + write("

No results available.

"); + } + write(DIV_CLOSE); + } - try{ - if(action.get(Tags.Desc)!=null) { - String escaped = StringEscapeUtils.escapeHtml(action.get(Tags.Desc)); - write(" || "+ escaped); - } - }catch(NullPointerException | NoSuchTagException e){} + private void composeMatchedResultTable(MatcherResult result) { + + for (ContentMatchResult contentResult : result.getResult()) { + // Based on the location try lookup the OCR matched entries. + Optional locationMatch = result.getLocationMatches().stream() + .filter(it -> it.location.location == contentResult.foundLocation).findFirst(); + if (locationMatch.isPresent()) { + LocationMatch it = locationMatch.get(); + write("
"); + write("
Expect: \"" + it.expectedElement._text + "\" at location: " + it.location.location + "\" location matched with:
"); + it.recognizedElements.forEach(recognizedElement -> + write("
\"" + recognizedElement._text + "\" at location: " + recognizedElement._location + "\" confidence: " + recognizedElement._confidence + "
")); + write("
"); + } - write(""); - if(actionPath.contains("./output")){ - actionPath = actionPath.replace("./output",".."); + // Write content match result. + write("" + ); + writeTableRow(() -> { + writeTableHeader("Result:"); + contentResult.expectedResult.getResult().forEach(it -> + writeTableCell(it.getMatchResult()) + ); + return true; + }); + writeTableRow(() -> { + writeTableHeader("Expected:"); + contentResult.expectedResult.getResult().forEach(it -> + writeTableCell(it.getCharacterMatch().getCharacter()) + ); + return true; + }); + writeTableRow(() -> { + writeTableHeader("Found:"); + contentResult.expectedResult.getResult().forEach(it -> + { + CharacterMatchEntry tmp = it.getCharacterMatch(); + writeTableCell(tmp.isMatched() ? tmp.getCounterPart().getCharacter() : ""); + } + ); + return true; + }); + write("
\"" + contentResult.expectedText + "\" " + + "[" + contentResult.totalMatched + "/" + contentResult.totalExpected + "] " + contentResult.matchedPercentage +"%
"); + if (contentResult.recognizedResult.getResult().stream().anyMatch(CharacterMatchEntry::isNotMatched)) { + write("

No match for OCR items:

"); + write("
    "); + contentResult.recognizedResult.getResult().forEach(it -> { + if (it.isNotMatched()) { + write("
  • " + it.getCharacter() + "
  • "); + } + }); + write("
"); + } } - write("

"); - write("
"); // Close executed action block container } - public void addTestVerdict(Verdict verdict){ - String verdictInfo = verdict.info(); - if(verdict.severity() > Verdict.OK.severity()) - verdictInfo = verdictInfo.replace(Verdict.OK.info(), ""); + private void writeTableHeader(T data) { + write("" + data + ""); + } - write("
"); // Open verdict block container - write("

Test verdict for this sequence: "+verdictInfo+"

"); - write("

Severity: "+verdict.severity()+"

"); - write("
"); // Close verdict block container + private void writeTableCell(T data) { + write("" + data + ""); + } - FINAL_VERDICT_FILENAME = verdict.verdictSeverityTitle(); + private void writeTableRow(Supplier data) { + write(""); + data.get(); + write(""); } public void close() { - write(""); // Close the main div container - for(String s:HTMLReporter.FOOTER){ - write(s); - } - out.close(); - - // Finally rename the HTML report by indicating the final Verdict - // sequence_1.html to sequence_1_OK.html - String htmlFilenameVerdict = htmlFilename.replace(".html", "_" + FINAL_VERDICT_FILENAME + ".html"); - new File(htmlFilename).renameTo(new File (htmlFilenameVerdict)); + write(DIV_CLOSE); // Close the main div container + for (String s : HTMLReporter.FOOTER) { + write(s); + } + out.close(); + + // Finally rename the HTML report by indicating the final Verdict + // sequence_1.html to sequence_1_OK.html + String htmlFilenameVerdict = htmlFilename.replace(".html", "_" + FINAL_VERDICT_FILENAME + ".html"); + new File(htmlFilename).renameTo(new File (htmlFilenameVerdict)); } private void write(String s) { diff --git a/testar/src/org/testar/reporting/Reporting.java b/testar/src/org/testar/reporting/Reporting.java index 68af292c8..145bb17bc 100644 --- a/testar/src/org/testar/reporting/Reporting.java +++ b/testar/src/org/testar/reporting/Reporting.java @@ -30,10 +30,12 @@ package org.testar.reporting; -import org.testar.monkey.alayer.State; import java.util.Set; +import org.checkerframework.checker.nullness.qual.Nullable; import org.testar.monkey.alayer.Action; +import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Verdict; +import org.testar.visualvalidation.matcher.MatcherResult; public interface Reporting { public void addSequenceStep(State state, String actionImagePath); @@ -42,5 +44,6 @@ public interface Reporting { public void addActionsAndUnvisitedActions(Set actions, Set concreteIdsOfUnvisitedActions); public void addSelectedAction(State state, Action action); public void addTestVerdict(Verdict verdict); + void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action); public void close(); } diff --git a/testar/src/org/testar/reporting/XMLSequenceReport.java b/testar/src/org/testar/reporting/XMLSequenceReport.java index 5243b9ece..3563a482f 100644 --- a/testar/src/org/testar/reporting/XMLSequenceReport.java +++ b/testar/src/org/testar/reporting/XMLSequenceReport.java @@ -36,11 +36,12 @@ import java.io.UnsupportedEncodingException; import java.util.Set; +import org.testar.OutputStructure; import org.testar.monkey.alayer.Action; import org.testar.monkey.alayer.State; import org.testar.monkey.alayer.Verdict; -import org.testar.OutputStructure; - +import org.testar.visualvalidation.matcher.MatcherResult; +import org.checkerframework.checker.nullness.qual.Nullable; @@ -156,6 +157,11 @@ public void addTestVerdict(Verdict verdict) { out.flush(); } + @Override + public void addVisualValidationResult(MatcherResult result, State state, @Nullable Action action) { + + } + private TestRun testRun = new TestRun(); @Override diff --git a/testar/src/org/testar/settingsdialog/SettingsDialog.java b/testar/src/org/testar/settingsdialog/SettingsDialog.java index 2eff7db4b..a1b90746a 100644 --- a/testar/src/org/testar/settingsdialog/SettingsDialog.java +++ b/testar/src/org/testar/settingsdialog/SettingsDialog.java @@ -36,6 +36,10 @@ import org.testar.settingsdialog.dialog.*; import org.testar.settingsdialog.dialog.StateModelPanel; +import org.testar.monkey.Pair; +import org.testar.monkey.Util; +import org.testar.monkey.alayer.exceptions.NoSuchTagException; +import org.testar.extendedsettings.ExtendedSettingFile; import org.testar.extendedsettings.ExtendedSettingsFactory; import javax.imageio.ImageIO; @@ -69,6 +73,7 @@ public class SettingsDialog extends JFrame implements Observer { private static final long serialVersionUID = 5156320008281200950L; static final String TESTAR_VERSION = "2.5.2 (27-Sep-2022)"; + static final String SETTINGS_FILENAME = "test.settings"; private String settingsFile; private Settings settings; @@ -183,6 +188,12 @@ private void checkSettings(Settings settings) throws IllegalStateException { throw new IllegalStateException("Temp Directory does not exist!"); } + try{ + settings.get(ConfigTags.ExtendedSettingsFile); + } catch (NoSuchTagException e){ + settings.set(ConfigTags.ExtendedSettingsFile, settingsFile.replace(SETTINGS_FILENAME, ExtendedSettingFile.FileName)); + } + settingPanels.forEach((k,v) -> v.right().checkSettings()); } diff --git a/testar/src/org/testar/settingsdialog/dialog/OraclePanel.java b/testar/src/org/testar/settingsdialog/dialog/OraclePanel.java index 6d3ac66b8..80f58f4f1 100644 --- a/testar/src/org/testar/settingsdialog/dialog/OraclePanel.java +++ b/testar/src/org/testar/settingsdialog/dialog/OraclePanel.java @@ -115,11 +115,10 @@ public OraclePanel() { enableWebConsoleWarningOracle.setToolTipText("Enable Web Console Warning Oracle"); add(enableWebConsoleWarningOracle); - // Disable the visualization until the implementation is ready - //enableVisualValidationCheckBox = new JCheckBox("Enable visual validation"); - //enableVisualValidationCheckBox.setBounds(10, 330, 180, 27); - //enableVisualValidationCheckBox.setToolTipText(ToolTipTexts.enableVisualValidationTTT); - //add(enableVisualValidationCheckBox); + enableVisualValidationCheckBox = new JCheckBox("Enable visual validation"); + enableVisualValidationCheckBox.setBounds(300, 300, 180, 27); + enableVisualValidationCheckBox.setToolTipText(ToolTipTexts.enableVisualValidationTTT); + add(enableVisualValidationCheckBox); freezeTimeLabel.setBounds(300, 330, 80, 27); add(freezeTimeLabel); @@ -149,8 +148,7 @@ public void populateFrom(final Settings settings) { txtWebConsoleWarningPattern.setText(settings.get(ConfigTags.WebConsoleWarningPattern)); // Visual validation elements VisualValidationSettings visualSetting = ExtendedSettingsFactory.createVisualValidationSettings(); - // Disable the visualization until the implementation is ready - //enableVisualValidationCheckBox.setSelected(visualSetting.enabled); + enableVisualValidationCheckBox.setSelected(visualSetting.enabled); } /** @@ -171,7 +169,6 @@ public void extractInformation(final Settings settings) { settings.set(ConfigTags.WebConsoleWarningPattern, txtWebConsoleWarningPattern.getText()); // Visual validation elements VisualValidationSettings visualSetting = ExtendedSettingsFactory.createVisualValidationSettings(); - // Disable the visualization until the implementation is ready - //visualSetting.enabled = enableVisualValidationCheckBox.isSelected(); + visualSetting.enabled = enableVisualValidationCheckBox.isSelected(); } } diff --git a/testar/src/org/testar/visualvalidation/DummyVisualValidator.java b/testar/src/org/testar/visualvalidation/DummyVisualValidator.java new file mode 100644 index 000000000..57bed7f8a --- /dev/null +++ b/testar/src/org/testar/visualvalidation/DummyVisualValidator.java @@ -0,0 +1,24 @@ +package org.testar.visualvalidation; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.monkey.alayer.AWTCanvas; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Widget; +import org.testar.visualvalidation.matcher.MatcherResult; + +public class DummyVisualValidator implements VisualValidationManager { + @Override + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + return new MatcherResult(); + } + + @Override + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { + return new MatcherResult(); + } + + @Override + public void Destroy() { + + } +} diff --git a/testar/src/org/testar/visualvalidation/Location.java b/testar/src/org/testar/visualvalidation/Location.java new file mode 100644 index 000000000..72ed44249 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/Location.java @@ -0,0 +1,21 @@ +package org.testar.visualvalidation; + +import java.awt.Rectangle; + +public class Location extends Rectangle { + public Location(int x, int y, int width, int height) { + this.x = x; + this.y = y; + this.width = width; + this.height = height; + } + + public Location() { + + } + + @Override + public String toString() { + return String.format("[x=%d, y=%d, width=%d, height=%d]", x, y, width, height); + } +} diff --git a/testar/src/org/testar/visualvalidation/TextElement.java b/testar/src/org/testar/visualvalidation/TextElement.java new file mode 100644 index 000000000..fd0a211ae --- /dev/null +++ b/testar/src/org/testar/visualvalidation/TextElement.java @@ -0,0 +1,34 @@ +package org.testar.visualvalidation; + +public class TextElement implements Comparable { + public final Location _location; + public final String _text; + + /** + * Constructor. + * + * @param location The relative location of the text inside the application. + * @param text The text. + */ + public TextElement(Location location, String text) { + _location = location; + _text = text; + } + + @Override + public String toString() { + return "TextElement{" + + "_location=" + _location + + ", _text='" + _text + '\'' + + '}'; + } + + @Override + public int compareTo(TextElement other) { + int result = -1; + if (_text.equals(other._text) && _location.equals(other._location)) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/VisualValidationFactory.java b/testar/src/org/testar/visualvalidation/VisualValidationFactory.java new file mode 100644 index 000000000..a4829dcb7 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/VisualValidationFactory.java @@ -0,0 +1,20 @@ +package org.testar.visualvalidation; + +import org.testar.extendedsettings.ExtendedSettingsFactory; + +public class VisualValidationFactory { + + public static VisualValidationManager createVisualValidator(String protocol) { + VisualValidationManager visualValidator; + VisualValidationSettings visualValidation = ExtendedSettingsFactory.createVisualValidationSettings(); + visualValidation.protocol = protocol; + + if (visualValidation.enabled) { + visualValidator = new VisualValidator(visualValidation); + } else { + visualValidator = new DummyVisualValidator(); + } + + return visualValidator; + } +} diff --git a/testar/src/org/testar/visualvalidation/VisualValidationManager.java b/testar/src/org/testar/visualvalidation/VisualValidationManager.java new file mode 100644 index 000000000..cbddad617 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/VisualValidationManager.java @@ -0,0 +1,33 @@ +package org.testar.visualvalidation; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.monkey.alayer.AWTCanvas; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Widget; +import org.testar.visualvalidation.matcher.MatcherResult; + +public interface VisualValidationManager { + /** + * Analyze the captured image and update the verdict. + * + * @param state The state of the application. + * @param screenshot The captured screenshot of the current state. + * @return The matching result of the expected text and the detected text on the captured image. + */ + MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot); + + /** + * Analyze the captured image and update the verdict. + * + * @param state The state of the application. + * @param screenshot The captured screenshot of the current state. + * @param widget Optional, the corresponding widget when the screenshot is an action shot. + * @return The matching result of the expected text and the detected text on the captured image. + */ + MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget); + + /** + * Destroy the visual validation manager. + */ + void Destroy(); +} diff --git a/testar/src/org/testar/visualvalidation/VisualValidationSettings.java b/testar/src/org/testar/visualvalidation/VisualValidationSettings.java index 29d45e025..45980ff72 100644 --- a/testar/src/org/testar/visualvalidation/VisualValidationSettings.java +++ b/testar/src/org/testar/visualvalidation/VisualValidationSettings.java @@ -31,6 +31,8 @@ package org.testar.visualvalidation; import org.testar.extendedsettings.ExtendedSettingBase; +import org.testar.visualvalidation.matcher.MatcherConfiguration; +import org.testar.visualvalidation.ocr.OcrConfiguration; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -40,11 +42,26 @@ @XmlAccessorType(XmlAccessType.FIELD) public class VisualValidationSettings extends ExtendedSettingBase { public Boolean enabled; + public OcrConfiguration ocrConfiguration; + public MatcherConfiguration matcherConfiguration; + // The selected protocol will be set automatically when we initialize TESTAR. + public String protocol; + + public static VisualValidationSettings CreateDefault() { + VisualValidationSettings defaultInstance = new VisualValidationSettings(); + defaultInstance.enabled = false; + defaultInstance.ocrConfiguration = OcrConfiguration.CreateDefault(); + defaultInstance.matcherConfiguration = MatcherConfiguration.CreateDefault(); + return defaultInstance; + } @Override public int compareTo(VisualValidationSettings other) { int res = -1; - if (this.enabled.equals(other.enabled)) { + if ((enabled.equals(other.enabled)) && + (ocrConfiguration.compareTo(other.ocrConfiguration) == 0) && + (matcherConfiguration.compareTo(other.matcherConfiguration) == 0) + ) { res = 0; } return res; @@ -54,12 +71,8 @@ public int compareTo(VisualValidationSettings other) { public String toString() { return "VisualValidationSettings{" + "enabled=" + enabled + + ", ocr=" + ocrConfiguration + + ", matcher=" + matcherConfiguration + '}'; } - - public static VisualValidationSettings CreateDefault() { - VisualValidationSettings DefaultInstance = new VisualValidationSettings(); - DefaultInstance.enabled = false; - return DefaultInstance; - } } diff --git a/testar/src/org/testar/visualvalidation/VisualValidationTag.java b/testar/src/org/testar/visualvalidation/VisualValidationTag.java new file mode 100644 index 000000000..994c36ebe --- /dev/null +++ b/testar/src/org/testar/visualvalidation/VisualValidationTag.java @@ -0,0 +1,9 @@ +package org.testar.visualvalidation; + +import org.testar.monkey.alayer.Tag; +import org.testar.monkey.alayer.TagsBase; +import org.testar.monkey.alayer.Verdict; + +public class VisualValidationTag extends TagsBase { + public static final Tag VisualValidationVerdict = from("VisualValidationVerdict", Verdict.class); +} diff --git a/testar/src/org/testar/visualvalidation/VisualValidationVerdict.java b/testar/src/org/testar/visualvalidation/VisualValidationVerdict.java new file mode 100644 index 000000000..b212e9f11 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/VisualValidationVerdict.java @@ -0,0 +1,31 @@ +package org.testar.visualvalidation; + +import org.testar.monkey.alayer.Verdict; +import org.testar.visualvalidation.matcher.ContentMatchResult; + +public class VisualValidationVerdict { + static final double warningSeverity = 0.1; + static final double errorSeverity = 0.15; + static final double failedToMatchSeverity = 0.16; + + private static String composeVerdictMessage(ContentMatchResult match) { + return String.format("\"%s\" matched %d%%.", match.expectedText, match.matchedPercentage); + } + + public static Verdict createSuccessVerdict(ContentMatchResult match) { + return new Verdict(Verdict.SEVERITY_OK, composeVerdictMessage(match)); + } + + public static Verdict createAlmostMatchedVerdict(ContentMatchResult match) { + return new Verdict(warningSeverity, composeVerdictMessage(match)); + } + + public static Verdict createHardlyMatchedVerdict(ContentMatchResult match) { + return new Verdict(errorSeverity, composeVerdictMessage(match)); + } + + public static Verdict createFailedToMatchVerdict(TextElement match) { + return new Verdict(failedToMatchSeverity, String.format("Failed to match \"%s\".", match._text)); + } + +} diff --git a/testar/src/org/testar/visualvalidation/VisualValidator.java b/testar/src/org/testar/visualvalidation/VisualValidator.java new file mode 100644 index 000000000..10e38ac3d --- /dev/null +++ b/testar/src/org/testar/visualvalidation/VisualValidator.java @@ -0,0 +1,252 @@ +package org.testar.visualvalidation; + +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.Logger; +import org.testar.monkey.alayer.AWTCanvas; +import org.testar.monkey.alayer.Action; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Tags; +import org.testar.monkey.alayer.Verdict; +import org.testar.monkey.alayer.Widget; +import org.testar.serialisation.ScreenshotSerialiser; +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.extractor.ExpectedTextCallback; +import org.testar.visualvalidation.extractor.ExtractorFactory; +import org.testar.visualvalidation.extractor.TextExtractorInterface; +import org.testar.visualvalidation.matcher.ContentMatchResult; +import org.testar.visualvalidation.matcher.MatcherResult; +import org.testar.visualvalidation.matcher.VisualMatcherFactory; +import org.testar.visualvalidation.matcher.VisualMatcherInterface; +import org.testar.visualvalidation.ocr.OcrConfiguration; +import org.testar.visualvalidation.ocr.OcrEngineFactory; +import org.testar.visualvalidation.ocr.OcrEngineInterface; +import org.testar.visualvalidation.ocr.OcrResultCallback; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.awt.Color; +import java.awt.Graphics2D; +import java.awt.Rectangle; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.testar.visualvalidation.VisualValidationTag.VisualValidationVerdict; +import static org.testar.visualvalidation.VisualValidationVerdict.createAlmostMatchedVerdict; +import static org.testar.visualvalidation.VisualValidationVerdict.createFailedToMatchVerdict; +import static org.testar.visualvalidation.VisualValidationVerdict.createHardlyMatchedVerdict; +import static org.testar.visualvalidation.VisualValidationVerdict.createSuccessVerdict; + +public class VisualValidator implements VisualValidationManager, OcrResultCallback, ExpectedTextCallback { + static final Color darkOrange = new Color(255, 128, 0); + private final String TAG = "VisualValidator"; + private final VisualMatcherInterface _matcher; + private final OcrEngineInterface _ocrEngine; + private final Object _ocrResultSync = new Object(); + private final AtomicBoolean _ocrResultReceived = new AtomicBoolean(); + private final TextExtractorInterface _extractor; + private final Object _expectedTextSync = new Object(); + private final AtomicBoolean _expectedTextReceived = new AtomicBoolean(); + private final VisualValidationSettings _settings; + private int analysisId = 0; + private MatcherResult _matcherResult = null; + private List _ocrItems = null; + private List _expectedText = null; + + public VisualValidator(@NonNull VisualValidationSettings settings) { + _settings = settings; + OcrConfiguration ocrConfig = _settings.ocrConfiguration; + if (ocrConfig.enabled) { + _ocrEngine = OcrEngineFactory.createOcrEngine(ocrConfig); + } else { + _ocrEngine = null; + } + + if (_settings.protocol.contains("webdriver_generic")) { + _extractor = ExtractorFactory.CreateExpectedTextExtractorWebdriver(); + } else { + _extractor = ExtractorFactory.CreateExpectedTextExtractorDesktop(); + } + + _matcher = VisualMatcherFactory.createLocationMatcher(_settings.matcherConfiguration); + } + + public static boolean isBetween(int x, int lower, int upper) { + return lower <= x && x <= upper; + } + + @Override + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot) { + return AnalyzeImage(state, screenshot, null); + } + + @Override + public MatcherResult AnalyzeImage(State state, @Nullable AWTCanvas screenshot, @Nullable Widget widget) { + // Create new session + startNewAnalysis(); + + if (screenshot != null) { + // Start ocr analysis, provide callback once finished. + parseScreenshot(screenshot); + + // Start extracting text, provide callback once finished. + extractExpectedText(state, widget); + + // Match the expected text with the detected text. + matchText(); + + // Calculate the verdict and create a screenshot with annotations which can be used by the HTML reporter. + processMatchResult(state, screenshot); + } else { + Logger.log(Level.ERROR, TAG, "No screenshot for current state, skipping visual validation"); + } + return _matcherResult; + } + + private void startNewAnalysis() { + analysisId++; + synchronized (_ocrResultSync) { + _ocrResultReceived.set(false); + } + synchronized (_expectedTextSync) { + _expectedTextReceived.set(false); + } + _matcherResult = null; + Logger.log(Level.INFO, TAG, "Starting new analysis {}", analysisId); + } + + private void parseScreenshot(AWTCanvas screenshot) { + _ocrEngine.AnalyzeImage(screenshot.image(), this); + } + + private void extractExpectedText(State state, @Nullable Widget widget) { + _extractor.ExtractExpectedText(state, widget, this); + } + + private void matchText() { + waitForResults(); + + _matcherResult = _matcher.Match(_ocrItems, _expectedText); + } + + private void waitForResult(@NonNull AtomicBoolean receivedFlag, Object syncObject) { + if (!receivedFlag.get()) { + //noinspection SynchronizationOnLocalVariableOrMethodParameter + synchronized (syncObject) { + try { + syncObject.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + private void waitForResults() { + waitForResult(_ocrResultReceived, _ocrResultSync); + waitForResult(_expectedTextReceived, _expectedTextSync); + } + + private void processMatchResult(State state, AWTCanvas screenshot) { + // Create a copy, so we can annotate the results without modifying the original one. + final AWTCanvas copy = new AWTCanvas( + screenshot.width(), + screenshot.height(), + screenshot.deepCopyImage(), + AWTCanvas.StorageFormat.PNG, + 1.0); + + final Verdict[] validationVerdict = {Verdict.OK}; + if (_matcherResult != null) { + // Draw a rectangle on each expected text element that has been recognized. + final ContentMatchResult[] lowestMatch = {null}; + _matcherResult.getResult().forEach(result -> { + java.awt.Color penColor; + Verdict verdict; + if (result.matchedPercentage == 100) { + penColor = java.awt.Color.green; + verdict = createSuccessVerdict(result); + } else if (isBetween(result.matchedPercentage, + _settings.matcherConfiguration.failedToMatchPercentageThreshold, + 99)) { + penColor = java.awt.Color.yellow; + verdict = createAlmostMatchedVerdict(result); + } else { + penColor = darkOrange; + verdict = createHardlyMatchedVerdict(result); + } + + // Store the result with the lowest percentage including the corresponding verdict. + if ((lowestMatch[0] == null) || + (lowestMatch[0].matchedPercentage > result.matchedPercentage)) { + lowestMatch[0] = result; + validationVerdict[0] = verdict; + } + + drawRectangle(copy, result.foundLocation, penColor); + } + ); + _matcherResult.getNoLocationMatches().forEach(result -> + { + if (result instanceof ExpectedElement) { + drawRectangle(copy, result._location, java.awt.Color.red); + // Overwrite the verdict if we couldn't find a location match for expected text. + validationVerdict[0] = createFailedToMatchVerdict(result); + } else { + drawRectangle(copy, result._location, Color.magenta); + } + } + ); + } + // Store the validation verdict for this run. + state.set(VisualValidationVerdict, validationVerdict[0]); + + // Store the annotated screenshot, so they can be used by the HTML report generator. + String stateId = state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"); + Action action = state.get(Tags.ExecutedAction, null); + if (action != null) { + String actionID = action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"); + ScreenshotSerialiser.saveActionshot(stateId, actionID + MatcherResult.ScreenshotPostFix, copy); + } else { + ScreenshotSerialiser.saveStateshot(stateId + MatcherResult.ScreenshotPostFix, copy); + } + + Logger.log(Level.INFO, TAG, "Processed match results"); + } + + private void drawRectangle(AWTCanvas screenshot, Rectangle location, java.awt.Color color) { + Graphics2D graphics = screenshot.image().createGraphics(); + graphics.setColor(color); + graphics.drawRect(location.x, location.y, location.width - 1, location.height - 1); + graphics.dispose(); + } + + @Override + public void Destroy() { + _matcher.destroy(); + _extractor.Destroy(); + if (_ocrEngine != null) { + _ocrEngine.Destroy(); + } + } + + @Override + public void reportResult(@NonNull List detectedText) { + synchronized (_ocrResultSync) { + _ocrItems = detectedText; + _ocrResultReceived.set(true); + _ocrResultSync.notifyAll(); + Logger.log(Level.INFO, TAG, "Received {} OCR result", detectedText.size()); + } + } + + @Override + public void ReportExtractedText(@NonNull List expectedText) { + synchronized (_expectedTextSync) { + _expectedText = expectedText; + _expectedTextReceived.set(true); + _expectedTextSync.notifyAll(); + Logger.log(Level.INFO, TAG, "Received {} expected result", expectedText.size()); + } + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/DummyExtractor.java b/testar/src/org/testar/visualvalidation/extractor/DummyExtractor.java new file mode 100644 index 000000000..6e028cb90 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/DummyExtractor.java @@ -0,0 +1,17 @@ +package org.testar.visualvalidation.extractor; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Widget; + +public class DummyExtractor implements TextExtractorInterface { + @Override + public void ExtractExpectedText(State state, @Nullable Widget widget, ExpectedTextCallback callback) { + + } + + @Override + public void Destroy() { + + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExpectedElement.java b/testar/src/org/testar/visualvalidation/extractor/ExpectedElement.java new file mode 100644 index 000000000..c615dbfcf --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExpectedElement.java @@ -0,0 +1,17 @@ +package org.testar.visualvalidation.extractor; + +import org.testar.visualvalidation.Location; +import org.testar.visualvalidation.TextElement; + +public class ExpectedElement extends TextElement { + + /** + * Constructor. + * + * @param location The relative location of the text inside the application. + * @param text The text. + */ + public ExpectedElement(Location location, String text) { + super(location, text); + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExpectedTextCallback.java b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextCallback.java new file mode 100644 index 000000000..e40c79180 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextCallback.java @@ -0,0 +1,18 @@ +package org.testar.visualvalidation.extractor; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.testar.monkey.alayer.Widget; + +import java.util.List; + +/** + * Callback function for sharing the expected text for all the {@link Widget}. + */ +public interface ExpectedTextCallback { + /** + * Report the expected text to the caller. + * + * @param expectedText The expected text for all the {@link Widget}. + */ + void ReportExtractedText(@NonNull List expectedText); +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java new file mode 100644 index 000000000..dcafda709 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorBase.java @@ -0,0 +1,227 @@ +package org.testar.visualvalidation.extractor; + + +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.Logger; +import org.testar.extendedsettings.ExtendedSettingsFactory; +import org.testar.monkey.Environment; +import org.testar.monkey.Util; +import org.testar.monkey.alayer.Roles; +import org.testar.monkey.alayer.Shape; +import org.testar.monkey.alayer.Tag; +import org.testar.monkey.alayer.Tags; +import org.testar.monkey.alayer.TagsBase; +import org.testar.monkey.alayer.Widget; +import org.testar.visualvalidation.Location; + +import java.awt.Rectangle; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.testar.monkey.alayer.Tags.Path; +import static org.testar.monkey.alayer.Tags.Role; +import static org.testar.monkey.alayer.Tags.Shape; + +public class ExpectedTextExtractorBase extends Thread implements TextExtractorInterface { + private static final String TAG = "ExpectedTextExtractor"; + /** + * Lookup table to map the name of the name of a {@link Tag}, to the actual {@link Tag}. + * Holds all the available tag's which could hold text (String types, only). The key represents the {@link Tag} in String type, value + */ + @SuppressWarnings("unchecked") + private static final Map> _tag = TagsBase.tagSet().stream() + .filter(tag -> tag.type().equals(String.class)) + .collect(Collectors.toMap(Tag::name, tag -> (Tag) tag)); + private final Object _threadSync = new Object(); + final private Tag defaultTag; + private final boolean _loggingEnabled; + /** + * A blacklist for {@link Widget}'s which should be ignored based on their {@code Role} because they don't contain + * readable text. Optional when the value (which represents the ancestor path) is set, the {@link Widget} should + * only be ignored when the ancestor path is equal with the {@link Widget} under investigation. + */ + private final Map> _blacklist = new HashMap<>(); + /** + * A lookup table which indicates based on the {@code Role} which {@link Tag} should be used to extract the text. + */ + private final Map _lookupTable = new HashMap<>(); + AtomicBoolean running = new AtomicBoolean(true); + private org.testar.monkey.alayer.State _state = null; + private ExpectedTextCallback _callback = null; + private Widget _widget = null; + + ExpectedTextExtractorBase(Tag defaultTag) { + WidgetTextConfiguration config = ExtendedSettingsFactory.createWidgetTextConfiguration(); + // Load the extractor configuration into a lookup table for quick access. + try { + config.widget.forEach(it -> { + if (it.ignore) { + List ancestor = it.ancestor.isEmpty() ? + Collections.emptyList() : Collections.singletonList(it.ancestor); + _blacklist.merge(it.role, ancestor, (list1, list2) -> + Stream.concat(list1.stream(), list2.stream()).collect(Collectors.toList())); + } else { + _lookupTable.put(it.role, it.tag); + } + }); + } catch (NullPointerException ignored) { + } + + _loggingEnabled = config.loggingEnabled; + + this.defaultTag = defaultTag; + + setName(TAG); + start(); + } + + static Rectangle getLocation(Widget widget) { + Shape dimension = widget.get(Shape, null); + + int x = dimension != null ? (int) dimension.x() : 0; + int y = dimension != null ? (int) dimension.y() : 0; + int width = dimension != null ? (int) dimension.width() : 0; + int height = dimension != null ? (int) dimension.height() : 0; + + return new Rectangle(x, y, width, height); + } + + @Override + public void ExtractExpectedText(org.testar.monkey.alayer.State state, @Nullable Widget widget, ExpectedTextCallback callback) { + synchronized (_threadSync) { + _state = state; + _widget = widget; + _callback = callback; + if (_loggingEnabled) { + Logger.log(Level.TRACE, TAG, "Queue new text extract."); + } + _threadSync.notifyAll(); + } + } + + @Override + public void run() { + while (running.get()) { + synchronized (_threadSync) { + try { + // Wait until we need to inspect a new image. + _threadSync.wait(); + if (!running.get()) { + break; + } + extractText(); + + } catch (InterruptedException e) { + Logger.log(Level.ERROR, TAG, "Wait interrupted"); + e.printStackTrace(); + } + } + } + } + + private void extractText() { + if (_state == null || _callback == null) { + Logger.log(Level.ERROR, TAG, "Should not try to extract text on empty state/callback"); + return; + } + + // Acquire the absolute location of the SUT on the screen. + Rectangle applicationPosition = null; + for (Widget widget : _state) { + String rootElementPath = "[0]"; + if (widget.get(Path).contentEquals(rootElementPath)) { + applicationPosition = getLocation(widget); + break; + } + } + Objects.requireNonNull(applicationPosition); + + double displayScale = Environment.getInstance().getDisplayScale(_state.get(Tags.HWND, (long) 0)); + List expectedElements = new ArrayList<>(); + if (_widget != null) { + extractedText(getLocation(_widget), displayScale, expectedElements, _widget); + } else { + Rectangle finalApplicationPosition = applicationPosition; + _state.forEach(widget -> extractedText(finalApplicationPosition, displayScale, expectedElements, widget)); + } + + _callback.ReportExtractedText(expectedElements); + } + + private void extractedText(Rectangle applicationPosition, double displayScale, List expectedElements, Widget widget) { + String widgetRole = widget.get(Role).name(); + String text = widget.get(getVisualTextTag(widgetRole), ""); + StringBuilder sb = new StringBuilder(); + Util.ancestors(widget).forEach(it -> sb.append(WidgetTextSetting.ANCESTOR_SEPARATOR).append(it.get(Role, Roles.Widget))); + String ancestors = sb.toString(); + boolean ignored = true; + + if (widgetIsIncluded(widget, widgetRole, ancestors)) { + if (text != null && !text.isEmpty()) { + Rectangle absoluteLocation = getLocation(widget); + Location relativeLocation = new Location( + (int) ((absoluteLocation.x - applicationPosition.x) * displayScale), + (int) ((absoluteLocation.y - applicationPosition.y) * displayScale), + (int) (absoluteLocation.width * displayScale), + (int) (absoluteLocation.height * displayScale)); + expectedElements.add(new ExpectedElement(relativeLocation, text)); + ignored = false; + } + } + + if (_loggingEnabled) { + Logger.log(Level.INFO, TAG, "Widget {} with role {} and ancestors {} is {}", + text, widgetRole, ancestors, ignored ? "ignored" : "added"); + } + } + + protected boolean widgetIsIncluded(Widget widget, String role, String ancestors) { + boolean containsReadableText = true; + if (_blacklist.containsKey(role)) { + containsReadableText = false; + try { + List blacklistedAncestors = _blacklist.get(role); + if (!blacklistedAncestors.isEmpty()) { + // Check if we should ignore this widget based on its ancestors. + containsReadableText = blacklistedAncestors.stream().noneMatch(ancestors::equals); + } + } catch (NullPointerException ignored) { + + } + } + return containsReadableText; + } + + private Tag getVisualTextTag(String widgetRole) { + return _tag.getOrDefault(_lookupTable.getOrDefault(widgetRole, ""), defaultTag); + } + + @Override + public void Destroy() { + stopAndJoinThread(); + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Extractor destroyed."); + } + } + + private void stopAndJoinThread() { + synchronized (_threadSync) { + running.set(false); + _threadSync.notifyAll(); + } + + try { + join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java new file mode 100644 index 000000000..f99ca81fd --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorDesktop.java @@ -0,0 +1,10 @@ +package org.testar.visualvalidation.extractor; + +import static org.testar.monkey.alayer.Tags.Title; + +public class ExpectedTextExtractorDesktop extends ExpectedTextExtractorBase { + + ExpectedTextExtractorDesktop() { + super(Title); + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java new file mode 100644 index 000000000..0f3dd7b55 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExpectedTextExtractorWebdriver.java @@ -0,0 +1,19 @@ +package org.testar.visualvalidation.extractor; + +import org.testar.monkey.alayer.Widget; + +import static org.testar.monkey.alayer.webdriver.enums.WdTags.WebIsFullOnScreen; +import static org.testar.monkey.alayer.webdriver.enums.WdTags.WebTextContent; + +class ExpectedTextExtractorWebdriver extends ExpectedTextExtractorBase { + + ExpectedTextExtractorWebdriver() { + super(WebTextContent); + } + + @Override + protected boolean widgetIsIncluded(Widget widget, String role, String ancestors) { + // Check if the widget is visible + return widget.get(WebIsFullOnScreen) && super.widgetIsIncluded(widget, role, ancestors); + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/ExtractorFactory.java b/testar/src/org/testar/visualvalidation/extractor/ExtractorFactory.java new file mode 100644 index 000000000..c37ed20fd --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/ExtractorFactory.java @@ -0,0 +1,15 @@ +package org.testar.visualvalidation.extractor; + +public class ExtractorFactory { + public static TextExtractorInterface CreateDummyExtractor() { + return new DummyExtractor(); + } + + public static TextExtractorInterface CreateExpectedTextExtractorDesktop() { + return new ExpectedTextExtractorDesktop(); + } + + public static TextExtractorInterface CreateExpectedTextExtractorWebdriver() { + return new ExpectedTextExtractorWebdriver(); + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/TextExtractorInterface.java b/testar/src/org/testar/visualvalidation/extractor/TextExtractorInterface.java new file mode 100644 index 000000000..f52519f54 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/TextExtractorInterface.java @@ -0,0 +1,21 @@ +package org.testar.visualvalidation.extractor; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.monkey.alayer.State; +import org.testar.monkey.alayer.Widget; + +public interface TextExtractorInterface { + /** + * Extract the expected text for all the available {@link Widget}'s in the given {@link State}. + * + * @param state The current state of the application under test. + * @param widget When set we only extract the text from this widget instead of the entire state. + * @param callback Callback function for returning the expected text. + */ + void ExtractExpectedText(State state, @Nullable Widget widget, ExpectedTextCallback callback); + + /** + * Destroy the text extractor. + */ + void Destroy(); +} diff --git a/testar/src/org/testar/visualvalidation/extractor/WidgetTextConfiguration.java b/testar/src/org/testar/visualvalidation/extractor/WidgetTextConfiguration.java new file mode 100644 index 000000000..82e3f657a --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/WidgetTextConfiguration.java @@ -0,0 +1,58 @@ +package org.testar.visualvalidation.extractor; + +import org.testar.extendedsettings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class WidgetTextConfiguration extends ExtendedSettingBase { + List widget; + boolean loggingEnabled; + + public static WidgetTextConfiguration CreateDefault() { + WidgetTextConfiguration instance = new WidgetTextConfiguration(); + + // Desktop protocol + WidgetTextSetting scrollBar = WidgetTextSetting.CreateIgnore("UIAScrollBar"); + WidgetTextSetting menuBar = WidgetTextSetting.CreateIgnore("UIAMenuBar"); + WidgetTextSetting statusBar = WidgetTextSetting.CreateIgnore("UIAStatusBar"); + WidgetTextSetting textEdit = WidgetTextSetting.CreateExtract("UIAEdit", "UIAValueValue"); + WidgetTextSetting icon = WidgetTextSetting.CreateIgnoreAncestorBased("UIAMenuItem", + Arrays.asList("UIAMenuBar", "UIATitleBar", "UIAWindow", "Process")); + WidgetTextSetting toolBarButtons = WidgetTextSetting.CreateIgnoreAncestorBased("UIAButton", + Arrays.asList("UIATitleBar", "UIAWindow", "Process")); + WidgetTextSetting scrollBarButtons = WidgetTextSetting.CreateIgnoreAncestorBased("UIAButton", + Arrays.asList("UIAScrollBar", "UIAEdit", "UIAWindow", "Process")); + + // Webdriver protocol + WidgetTextSetting skipToContentWdou = WidgetTextSetting.CreateIgnoreAncestorBased("WdA", + Arrays.asList("WdDIV", "Process")); + + instance.widget = new ArrayList<>(Arrays.asList(scrollBar, statusBar, menuBar, textEdit, icon, toolBarButtons, scrollBarButtons, skipToContentWdou)); + instance.loggingEnabled = false; + return instance; + } + + @Override + public int compareTo(WidgetTextConfiguration other) { + int result = -1; + + if (widget.size() == other.widget.size() && loggingEnabled == other.loggingEnabled) { + for (int i = 0; i < widget.size(); i++) { + int index = i; + if (other.widget.stream().noneMatch(it -> it.compareTo(widget.get(index)) == 0)) { + return result; + } + } + return 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/extractor/WidgetTextSetting.java b/testar/src/org/testar/visualvalidation/extractor/WidgetTextSetting.java new file mode 100644 index 000000000..bfdc58a52 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/extractor/WidgetTextSetting.java @@ -0,0 +1,58 @@ +package org.testar.visualvalidation.extractor; + +import org.testar.extendedsettings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.List; + +import static java.util.Collections.emptyList; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +class WidgetTextSetting extends ExtendedSettingBase { + public static final boolean IGNORE = true; + public static final String ANCESTOR_SEPARATOR = "::"; + + String role; + String tag; + Boolean ignore; + String ancestor; + + private static WidgetTextSetting CreateRaw(String role, Boolean ignore, String tag, List ancestor) { + WidgetTextSetting result = new WidgetTextSetting(); + result.role = role; + result.tag = tag; + result.ignore = ignore; + StringBuilder sb = new StringBuilder(); + ancestor.forEach(it -> sb.append(ANCESTOR_SEPARATOR).append(it)); + result.ancestor = sb.toString(); + return result; + } + + public static WidgetTextSetting CreateIgnoreAncestorBased(String role, List ancestor) { + return CreateRaw(role, IGNORE, "", ancestor); + } + + public static WidgetTextSetting CreateIgnore(String role) { + return CreateRaw(role, IGNORE, "", emptyList()); + } + + public static WidgetTextSetting CreateExtract(String role, String tag) { + return CreateRaw(role, false, tag, emptyList()); + } + + @Override + public int compareTo(WidgetTextSetting other) { + int result = -1; + if (role.contentEquals(other.role) + && (tag.contentEquals(other.tag)) + && (ignore.equals(other.ignore) + && (ancestor.equals(other.ancestor))) + ) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/CharacterMatch.java b/testar/src/org/testar/visualvalidation/matcher/CharacterMatch.java new file mode 100644 index 000000000..78f66578e --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/CharacterMatch.java @@ -0,0 +1,41 @@ +package org.testar.visualvalidation.matcher; + +/** + * Stores the outcome of the matcher result for the given character. + */ +public class CharacterMatch { + final CharacterMatchEntry character; + CharacterMatchResult result; + + /** + * Constructor. + * + * @param character The character which we try to match. + */ + public CharacterMatch(char character) { + result = CharacterMatchResult.NO_MATCH; + this.character = new CharacterMatchEntry(character); + } + + /** + * @return Get the result of the character match. + */ + public CharacterMatchResult getMatchResult() { + return result; + } + + /** + * @return Get the character match. + */ + public CharacterMatchEntry getCharacterMatch() { + return character; + } + + @Override + public String toString() { + return "CharacterMatch{" + + "result=" + result + + ", ch=" + character + + '}'; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/CharacterMatchEntry.java b/testar/src/org/testar/visualvalidation/matcher/CharacterMatchEntry.java new file mode 100644 index 000000000..e330bac5e --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/CharacterMatchEntry.java @@ -0,0 +1,71 @@ +package org.testar.visualvalidation.matcher; + +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * Container to store the matched counterpart of the expected character. + */ +public class CharacterMatchEntry { + final Character character; + CharacterMatchEntry match; + + /** + * Constructor. + * + * @param character The character which we try to match. + */ + CharacterMatchEntry(char character) { + this.character = character; + match = null; + } + + /** + * Mark this entry as matched. + * + * @param matchedCharacter The matched character. + */ + public void Match(@NonNull CharacterMatchEntry matchedCharacter) { + match = matchedCharacter; + matchedCharacter.match = this; + } + + /** + * @return Get the character. + */ + public Character getCharacter() { + return character; + } + + /** + * @return Get the matched counterpart of this character. + */ + public CharacterMatchEntry getCounterPart() { + return match; + } + + /** + * Check if the character has not matched. + * + * @return True when not matched. + */ + public boolean isNotMatched() { + return match == null; + } + + /** + * Check if the character is matched. + * + * @return True when matched. + */ + public boolean isMatched() { + return !isNotMatched(); + } + + @Override + public String toString() { + return "CharacterMatchEntry{" + + "match=" + Integer.toHexString(System.identityHashCode(match)) + + ", character=" + character + + '}'; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/CharacterMatchResult.java b/testar/src/org/testar/visualvalidation/matcher/CharacterMatchResult.java new file mode 100644 index 000000000..8e5ab4213 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/CharacterMatchResult.java @@ -0,0 +1,27 @@ +package org.testar.visualvalidation.matcher; + +/** + * The match result of an individual character. + */ +public enum CharacterMatchResult { + MATCHED, + CASE_MISMATCH, + WHITESPACE_CORRECTED, + NO_MATCH; + + @Override + public String toString() { + switch (this) { + case MATCHED: + return "V"; + case CASE_MISMATCH: + return "C"; + case WHITESPACE_CORRECTED: + return "W"; + case NO_MATCH: + return "X"; + default: + return ""; + } + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/ContentMatchResult.java b/testar/src/org/testar/visualvalidation/matcher/ContentMatchResult.java new file mode 100644 index 000000000..65ad445a0 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/ContentMatchResult.java @@ -0,0 +1,91 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.Location; + +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * Holds the matching result for the content of the expected text. + */ +public class ContentMatchResult implements Comparable { + public final ExpectedTextMatchResult expectedResult; + public final RecognizedTextMatchResult recognizedResult; + public final String expectedText; + public final Location foundLocation; + public final long totalMatched; + public final long totalExpected; + public final int matchedPercentage; + + public ContentMatchResult(ExpectedTextMatchResult expectedResult, RecognizedTextMatchResult recognizedResult, Location foundLocation) { + this.expectedResult = expectedResult; + this.recognizedResult = recognizedResult; + this.foundLocation = foundLocation; + totalMatched = expectedResult.expectedText.stream() + .filter(e -> e.result != CharacterMatchResult.NO_MATCH) + .count(); + totalExpected = expectedResult.expectedText.size(); + matchedPercentage = Math.round((float) totalMatched * 100 / totalExpected); + expectedText = expectedResult.expectedText.stream(). + map(e -> e.character.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)); + } + + @Override + public String toString() { + StringBuilder str = new StringBuilder() + .append("\n") + + .append("Matched \"").append(expectedText) + .append("\"\n") + + .append("Result: ") + .append(expectedResult.expectedText.stream().map(it -> it.result).collect(Collectors.toList())) + .append(" [").append(totalMatched).append("/").append(totalExpected).append("]\n") + + .append("Expect: ").append(expectedResult.expectedText.stream() + .map(it -> it.character.character) + .collect(Collectors.toList())) + .append("\n") + + .append("Found: ") + .append(expectedResult.expectedText.stream() + .map(it -> { + if (it.character.isMatched()) { + return it.character.match.character; + } else { + return " "; + } + }) + .collect(Collectors.toList())) + .append("\n"); + + List garbage = recognizedResult.recognized.stream() + .filter(CharacterMatchEntry::isNotMatched) + .collect(Collectors.toList()); + + if (!garbage.isEmpty()) { + str.append("Garbage:"); + str.append(garbage.stream() + .map(it -> it.character) + .collect(Collectors.toList())); + } + + return str.toString(); + } + + @Override + public int compareTo(ContentMatchResult other) { + int result = -1; + if (expectedText.equals(other.expectedText) && + (totalExpected == other.totalExpected) && + (totalMatched == other.totalMatched)) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/ContentMatcher.java b/testar/src/org/testar/visualvalidation/matcher/ContentMatcher.java new file mode 100644 index 000000000..2b36f5197 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/ContentMatcher.java @@ -0,0 +1,163 @@ +package org.testar.visualvalidation.matcher; + +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.testar.Logger; +import org.testar.monkey.Pair; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +class ContentMatcher { + static final String TAG = "ContentMatcher"; + + /** + * Tries to match characters from the recognized text with the expected text characters. Recognized text elements + * are first sorted based on their location. By calculating the line height of text the recognized text elements are + * first grouped per text line. After that they are sorted based on their x coordinate. The result of this sorting + * mechanism is a grid from left to right and top to bottom. The algorithm tries to find a matching character from + * the grid, once found the index is stored and will be used as the starting position for the next iteration. + */ + public static ContentMatchResult Match(LocationMatch locationMatch, MatcherConfiguration configuration) { + String expectedText = locationMatch.expectedElement._text; + ExpectedTextMatchResult expectedResult = new ExpectedTextMatchResult(expectedText); + if (configuration.loggingEnabled) { + Logger.log(Level.INFO, TAG, + "Expected text @{} : {}", + locationMatch.expectedElement._location, + expectedResult); + } + + // Sort the recognized elements. + List sorted = sortRecognizedElements(locationMatch, configuration); + RecognizedTextMatchResult recognizedResult = new RecognizedTextMatchResult(sorted); + if (configuration.loggingEnabled) { + Logger.log(Level.INFO, TAG, "Recognized text: {}", recognizedResult); + } + + // Iterate over the expected text and try to match with a recognized element character. + int unMatchedSize = recognizedResult.recognized.size(); + int indexCounter = 0; + for (CharacterMatch expectedChar : expectedResult.expectedText) { + char actual = expectedChar.character.character; + + // Iterate over the recognized characters. + for (int k = indexCounter; k < unMatchedSize; k++) { + CharacterMatchEntry item = recognizedResult.recognized.get(k); + + // Try to match the actual char case-sensitive: + if (item.character == actual && item.isNotMatched()) { + expectedChar.character.Match(item); + expectedChar.result = CharacterMatchResult.MATCHED; + // We have found a match move the index counter. + indexCounter = k + 1; + break; + } + + // Try to match the actual char as case-insensitive: + if (Character.isLetter(actual)) { + int CASING = 32; + if (Math.abs(item.character.compareTo(actual)) == CASING && item.isNotMatched()) { + expectedChar.character.Match(item); + expectedChar.result = CharacterMatchResult.CASE_MISMATCH; + // We have found a match inside the unMatched List move one position. + indexCounter = k + 1; + break; + } + } + } + } + + correctWhitespaces(expectedResult); + + return new ContentMatchResult(expectedResult, recognizedResult, locationMatch.expectedElement._location); + } + + @NonNull + static List sortRecognizedElements(LocationMatch locationMatch, + MatcherConfiguration configuration) { + Map> lineBucket = + sortRecognizedElementsPerTextLine(locationMatch, configuration); + + // Get the sorted Y-axe coordinates for the identified lines. + List bucketSorted = lineBucket.keySet().stream().sorted().collect(Collectors.toList()); + + // For each line sort the elements based on their X-axis coordinate and add to the result. + List sorted = new ArrayList<>(); + bucketSorted.forEach(line -> + { + List sort = lineBucket.get(line).stream() + .sorted(Comparator.comparingInt(o -> o._location.x)) + .collect(Collectors.toList()); + sorted.addAll(sort); + } + ); + return sorted; + } + + @NonNull + static Map> sortRecognizedElementsPerTextLine(LocationMatch locationMatch, + MatcherConfiguration configuration) { + // Determine the average height of the recognized text elements. + double lineHeight = locationMatch.recognizedElements.stream() + .mapToInt(e -> e._location.height) + .average() + .orElse(Double.NaN); + + // Create an overview of pairs reflecting the center line of the text and the recognized element. + List> lines = locationMatch.recognizedElements.stream() + .map(e -> new Pair<>(e._location.y + (e._location.height / 2), e)) + .collect(Collectors.toList()); + + final int margin = (int) Math.round(lineHeight / 2); + Map> lineBucket = new HashMap<>(); + + // Find a matching line within the margins of the center line, if not found create a new line. + for (Pair line : lines) { + boolean foundBucket = false; + for (Integer c : lineBucket.keySet()) { + if (IntStream.rangeClosed(c - margin, c + margin) + .boxed() + .collect(Collectors.toList()) + .contains(line.left())) { + if (configuration.loggingEnabled) { + Logger.log(Level.DEBUG, TAG, + "Line {} {} in range of bucket {}", + line.left(), + line.right(), + c); + } + lineBucket.get(c).add(line.right()); + foundBucket = true; + break; + } + } + if (!foundBucket) { + if (configuration.loggingEnabled) { + Logger.log(Level.DEBUG, TAG, + "No bucket found creating new one for {} {}", + line.left(), + line.right()); + } + lineBucket.put(line.left(), new ArrayList<>()); + lineBucket.get(line.left()).add(line.right()); + } + } + return lineBucket; + } + + static void correctWhitespaces(ExpectedTextMatchResult expectedResult) { + // Whitespaces are automatically corrected. + for (CharacterMatch res : expectedResult.expectedText) { + if (res.result == CharacterMatchResult.NO_MATCH && Character.isWhitespace(res.character.character)) { + res.result = CharacterMatchResult.WHITESPACE_CORRECTED; + } + } + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/ExpectedTextMatchResult.java b/testar/src/org/testar/visualvalidation/matcher/ExpectedTextMatchResult.java new file mode 100644 index 000000000..17418bad3 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/ExpectedTextMatchResult.java @@ -0,0 +1,43 @@ +package org.testar.visualvalidation.matcher; + +import java.util.ArrayList; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * The matcher result for the expected text. + */ +public class ExpectedTextMatchResult { + ArrayList expectedText; + + /** + * Constructor. + * + * @param expectedText The expected text that we are matching. + */ + public ExpectedTextMatchResult(String expectedText) { + this.expectedText = new ArrayList<>(expectedText.length()); + for (char ch : expectedText.toCharArray()) { + this.expectedText.add(new CharacterMatch(ch)); + } + } + + /** + * @return The matching result of the expected text characters. + */ + public ArrayList getResult() { + return expectedText; + } + + @Override + public String toString() { + return "ExpectedTextMatchResult (" + + expectedText.stream().map(e -> e.character.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)) + + ":" + expectedText.size() + "){ " + + expectedText.stream().map(CharacterMatch::toString).collect(Collectors.joining()); + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/LocationMatch.java b/testar/src/org/testar/visualvalidation/matcher/LocationMatch.java new file mode 100644 index 000000000..c07c58fa1 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/LocationMatch.java @@ -0,0 +1,41 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.Set; +import java.util.TreeSet; + +public class LocationMatch implements Comparable { + final public MatchLocation location; + final public ExpectedElement expectedElement; + + public Set recognizedElements = new TreeSet<>(); + + public LocationMatch(ExpectedElement expectedElement, int margin) { + this.location = new MatchLocation(margin, expectedElement._location); + this.expectedElement = expectedElement; + } + + public void addRecognizedElement(RecognizedElement element) { + recognizedElements.add(element); + } + + @Override + public String toString() { + return "LocationMatch{" + + "location=" + location + + ", expectedElement=" + expectedElement + + ", recognizedElements=" + recognizedElements + + '}'; + } + + @Override + public int compareTo(LocationMatch other) { + int result = -1; + if (location.equals(other.location) && expectedElement.equals(other.expectedElement)) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/LocationMatcher.java b/testar/src/org/testar/visualvalidation/matcher/LocationMatcher.java new file mode 100644 index 000000000..6e1e5c65b --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/LocationMatcher.java @@ -0,0 +1,123 @@ +package org.testar.visualvalidation.matcher; + +import org.apache.logging.log4j.Level; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.testar.Logger; +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.awt.Dimension; +import java.awt.Rectangle; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +public class LocationMatcher implements VisualMatcherInterface { + static final String TAG = "Matcher"; + private final MatcherConfiguration config; + + public LocationMatcher(MatcherConfiguration setting) { + config = setting; + } + + @Override + public void destroy() { + + } + + /** + * Match the recognized text elements with the expected text elements based on their location. For each expected + * text element we first link the related recognized text elements based on their location. This process is executed + * based on the surface area of the expected text in the order from small to large. This prevents that we don't + * accidentally map a recognized text element to a different expected element that may overlap the actual expected + * text element. Once we have finalized the linking we start the matching of the actual content. + * + * @param recognizedElements A list of recognized elements. + * @param expectedElements A list of expected text elements. + * @return The result of the matching algorithm. + */ + @Override + public MatcherResult Match(List recognizedElements, List expectedElements) { + // 1) Sort based on the area size of the element, from small to big + expectedElements.sort((element1, element2) -> { + Dimension element1Size = element1._location.getSize(); + Dimension element2Size = element2._location.getSize(); + return ((element1Size.width * element1Size.height) - (element2Size.width * element2Size.height)); + }); + + MatcherResult _matchResult = new MatcherResult(); + List _recognizedElements = new ArrayList<>(recognizedElements); + // 2) Match the OCR results based on their location. + expectedElements.forEach(expectedElement -> { + final LocationMatch locationMatch = getIntersectedElements(_recognizedElements, expectedElement); + + // All the recognized elements which lay inside the area of interest are linked to this match. + if (locationMatch != null) { + _matchResult.addLocationMatch(locationMatch); + + // 3) Now that we have collected all the recognized elements, try to match the content. + _matchResult.addContentMatchResult(ContentMatcher.Match(locationMatch, config)); + + // Remove all the recognized items that have been linked to this match from the list with items which + // we still need to match based on their location + locationMatch.recognizedElements.forEach(_recognizedElements::remove); + } else { + _matchResult.addNoLocationMatch(expectedElement); + } + }); + + // Compose the final matching result. + setUnmatchedElements(_matchResult, _recognizedElements); + return _matchResult; + } + + /** + * Find all unmatched elements and update the matcher result. + */ + void setUnmatchedElements(MatcherResult matcherResult, List recognizedElements) { + Set removeRecognizedList = new TreeSet<>(); + matcherResult.getLocationMatches().forEach(it -> removeRecognizedList.addAll(it.recognizedElements)); + + // Remove the elements which have been matched with expected elements. + removeRecognizedList.forEach(recognizedElements::remove); + + // The remainder of the recognized elements can be considered to be unmatched. + recognizedElements.forEach(matcherResult::addNoLocationMatch); + + if (config.loggingEnabled) { + Logger.log(Level.INFO, TAG, "No location match for the following detected text elements:\n {}", matcherResult.getNoLocationMatches().stream() + .map(e -> e._text + " " + e._location + "\n") + .collect(Collectors.joining())); + } + } + + /** + * Find all recognized text elements which intersect with the expected text element. + */ + @Nullable + LocationMatch getIntersectedElements(List recognizedElements, ExpectedElement expectedElement) { + final LocationMatch[] locationMatches = {null}; + recognizedElements.forEach(it -> { + final int margin = config.locationMatchMargin; + Rectangle areaOfInterest = it._location; + areaOfInterest.setBounds( + areaOfInterest.x - margin, + areaOfInterest.y - margin, + areaOfInterest.width + (2 * margin), + areaOfInterest.height + (2 * margin) + ); + + // If the recognized element lay inside the area of interest. + if (areaOfInterest.intersects(expectedElement._location)) { + // Prepare to make a match. + if (locationMatches[0] == null) { + locationMatches[0] = new LocationMatch(expectedElement, margin); + } + locationMatches[0].addRecognizedElement(it); + } + }); + return locationMatches[0]; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/MatchLocation.java b/testar/src/org/testar/visualvalidation/matcher/MatchLocation.java new file mode 100644 index 000000000..c34ab7d46 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/MatchLocation.java @@ -0,0 +1,21 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.Location; + +public class MatchLocation { + final public int margin; + final public Location location; + + public MatchLocation(int margin, Location location) { + this.margin = margin; + this.location = location; + } + + @Override + public String toString() { + return "MatchLocation{" + + "margin=" + margin + + ", location=" + location + + '}'; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/MatcherConfiguration.java b/testar/src/org/testar/visualvalidation/matcher/MatcherConfiguration.java new file mode 100644 index 000000000..e91ed0929 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/MatcherConfiguration.java @@ -0,0 +1,41 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.extendedsettings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +@XmlAccessorType(XmlAccessType.FIELD) +public class MatcherConfiguration extends ExtendedSettingBase { + public Integer locationMatchMargin; + public Integer failedToMatchPercentageThreshold; + boolean loggingEnabled; + + public static MatcherConfiguration CreateDefault() { + MatcherConfiguration instance = new MatcherConfiguration(); + instance.locationMatchMargin = 0; + instance.loggingEnabled = false; + instance.failedToMatchPercentageThreshold = 75; + return instance; + } + + @Override + public String toString() { + return "OcrConfiguration{" + + "margin=" + locationMatchMargin + + '}'; + } + + @Override + public int compareTo(MatcherConfiguration other) { + int result = -1; + if (locationMatchMargin.equals(other.locationMatchMargin) && + (loggingEnabled == other.loggingEnabled) && + (Objects.equals(failedToMatchPercentageThreshold, other.failedToMatchPercentageThreshold)) + ) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/MatcherResult.java b/testar/src/org/testar/visualvalidation/matcher/MatcherResult.java new file mode 100644 index 000000000..7a6385594 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/MatcherResult.java @@ -0,0 +1,50 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.TextElement; + +import java.util.Set; +import java.util.TreeSet; + +public class MatcherResult { + /** + * Postfix for the screenshots created by the visual validation module. + * Should be added to the end of normal action and state screenshots. E.g. "filename+postfix.extension" + */ + public static final String ScreenshotPostFix = "_vv"; + private final Set noMatches = new TreeSet<>(); + private final Set locationMatches = new TreeSet<>(); + private final Set contentMatchResults = new TreeSet<>(); + + public void addContentMatchResult(ContentMatchResult result) { + contentMatchResults.add(result); + } + + public void addLocationMatch(LocationMatch locationMatch) { + locationMatches.add(locationMatch); + } + + public void addNoLocationMatch(TextElement element) { + noMatches.add(element); + } + + public Set getNoLocationMatches() { + return noMatches; + } + + public Set getLocationMatches() { + return locationMatches; + } + + public Set getResult() { + return contentMatchResults; + } + + @Override + public String toString() { + return "MatcherResult{" + + "noLocationMatches(" + noMatches.size() + ")=" + noMatches + + ", locationMatches(" + locationMatches.size() + ")=" + locationMatches + + ", contentMatches(" + contentMatchResults.size() + ")=" + contentMatchResults + + '}'; + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/RecognizedTextMatchResult.java b/testar/src/org/testar/visualvalidation/matcher/RecognizedTextMatchResult.java new file mode 100644 index 000000000..9a4ed1f58 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/RecognizedTextMatchResult.java @@ -0,0 +1,45 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * The matcher result for the recognized text. + */ +public class RecognizedTextMatchResult { + List recognized; + + /** + * Constructor creates a consecutive order of recognized text characters. + * + * @param recognizedElements A list with recognized elements that should be treated as one. + */ + public RecognizedTextMatchResult(List recognizedElements) { + recognized = recognizedElements.stream().flatMap( + recognizedElement -> recognizedElement._text.chars().boxed().map(it -> Character.toChars(it)[0])) + .map(CharacterMatchEntry::new) + .collect(Collectors.toList()); + } + + /** + * @return The matching result of the recognized text characters. + */ + public List getResult() { + return recognized; + } + + @Override + public String toString() { + return "RecognizedTextMatchResult{" + + recognized.stream().map(e -> e.character).collect(Collector.of( + StringBuilder::new, + StringBuilder::append, + StringBuilder::append, + StringBuilder::toString)) + + ":" + recognized.size() + "){ " + + recognized.stream().map(CharacterMatchEntry::toString).collect(Collectors.joining()); + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/VisualDummyMatcher.java b/testar/src/org/testar/visualvalidation/matcher/VisualDummyMatcher.java new file mode 100644 index 000000000..882aaaa4b --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/VisualDummyMatcher.java @@ -0,0 +1,18 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; + +public class VisualDummyMatcher implements VisualMatcherInterface { + @Override + public MatcherResult Match(List ocrResult, List expectedText) { + return new MatcherResult(); + } + + @Override + public void destroy() { + + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/VisualMatcherFactory.java b/testar/src/org/testar/visualvalidation/matcher/VisualMatcherFactory.java new file mode 100644 index 000000000..de0261721 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/VisualMatcherFactory.java @@ -0,0 +1,11 @@ +package org.testar.visualvalidation.matcher; + +public class VisualMatcherFactory { + public static VisualMatcherInterface createDummyMatcher() { + return new VisualDummyMatcher(); + } + + public static VisualMatcherInterface createLocationMatcher(MatcherConfiguration setting) { + return new LocationMatcher(setting); + } +} diff --git a/testar/src/org/testar/visualvalidation/matcher/VisualMatcherInterface.java b/testar/src/org/testar/visualvalidation/matcher/VisualMatcherInterface.java new file mode 100644 index 000000000..faee2ce60 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/matcher/VisualMatcherInterface.java @@ -0,0 +1,12 @@ +package org.testar.visualvalidation.matcher; + +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.List; + +public interface VisualMatcherInterface { + MatcherResult Match(List ocrResult, List expectedText); + + void destroy(); +} diff --git a/testar/src/org/testar/visualvalidation/ocr/OcrConfiguration.java b/testar/src/org/testar/visualvalidation/ocr/OcrConfiguration.java new file mode 100644 index 000000000..1b7b2de89 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/OcrConfiguration.java @@ -0,0 +1,39 @@ +package org.testar.visualvalidation.ocr; + +import org.testar.extendedsettings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class OcrConfiguration extends ExtendedSettingBase { + public static final String TESSERACT_ENGINE = "tesseract"; + + public Boolean enabled; + public String engine; + + public static OcrConfiguration CreateDefault() { + OcrConfiguration instance = new OcrConfiguration(); + instance.enabled = true; + instance.engine = TESSERACT_ENGINE; + return instance; + } + + @Override + public String toString() { + return "OcrConfiguration{" + + "enabled=" + enabled + + ", engine='" + engine + '\'' + + '}'; + } + + @Override + public int compareTo(OcrConfiguration other) { + int result = -1; + if ((enabled.equals(other.enabled)) && + (engine.contentEquals(other.engine))) { + result = 0; + } + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/OcrEngineFactory.java b/testar/src/org/testar/visualvalidation/ocr/OcrEngineFactory.java new file mode 100644 index 000000000..7f5cae4eb --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/OcrEngineFactory.java @@ -0,0 +1,23 @@ +package org.testar.visualvalidation.ocr; + +import org.testar.visualvalidation.ocr.dummy.DummyOcrEngine; +import org.testar.visualvalidation.ocr.tesseract.TesseractOcrEngine; + +public class OcrEngineFactory { + + public static OcrEngineInterface createOcrEngine(OcrConfiguration settings) { + if (settings.engine.contentEquals(OcrConfiguration.TESSERACT_ENGINE)) { + return createTesseractEngine(); + } else { + return createDummyEngine(); + } + } + + static OcrEngineInterface createTesseractEngine() { + return new TesseractOcrEngine(); + } + + static OcrEngineInterface createDummyEngine() { + return new DummyOcrEngine(); + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/OcrEngineInterface.java b/testar/src/org/testar/visualvalidation/ocr/OcrEngineInterface.java new file mode 100644 index 000000000..4ecac6602 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/OcrEngineInterface.java @@ -0,0 +1,18 @@ +package org.testar.visualvalidation.ocr; + +import java.awt.image.BufferedImage; + +public interface OcrEngineInterface { + /** + * Analyze the given image with an OCR engine and return the detected text via the callback. + * + * @param image The image we want to analyze. + * @param callback Callback function for returning the detected text. + */ + void AnalyzeImage(BufferedImage image, OcrResultCallback callback); + + /** + * Destroy the OCR engine. + */ + void Destroy(); +} diff --git a/testar/src/org/testar/visualvalidation/ocr/OcrResultCallback.java b/testar/src/org/testar/visualvalidation/ocr/OcrResultCallback.java new file mode 100644 index 000000000..bb609ff0a --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/OcrResultCallback.java @@ -0,0 +1,15 @@ +package org.testar.visualvalidation.ocr; + +import java.util.List; + +/** + * Callback function for sharing the detected text by an OCR engine. + */ +public interface OcrResultCallback { + /** + * Report the detected text to the caller. + * + * @param detectedText The detected text by the OCR engine. + */ + void reportResult(List detectedText); +} diff --git a/testar/src/org/testar/visualvalidation/ocr/RecognizedElement.java b/testar/src/org/testar/visualvalidation/ocr/RecognizedElement.java new file mode 100644 index 000000000..b3dadda2a --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/RecognizedElement.java @@ -0,0 +1,32 @@ +package org.testar.visualvalidation.ocr; + +import org.testar.visualvalidation.Location; +import org.testar.visualvalidation.TextElement; + +/** + * A discovered text element by an OCR engine. + */ +public class RecognizedElement extends TextElement { + public final float _confidence; + + /** + * Constructor. + * + * @param location The relative location of the text inside the application. + * @param confidence The confidence level of the discovered text. + * @param text The discovered text. + */ + public RecognizedElement(Location location, float confidence, String text) { + super(location, text); + _confidence = confidence; + } + + @Override + public String toString() { + return "RecognizedElement{" + + "_location=" + _location + + ", _confidence=" + _confidence + + ", _text='" + _text + '\'' + + '}'; + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java b/testar/src/org/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java new file mode 100644 index 000000000..5d4ae590c --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/dummy/DummyOcrEngine.java @@ -0,0 +1,20 @@ +package org.testar.visualvalidation.ocr.dummy; + +import org.testar.visualvalidation.ocr.OcrEngineInterface; +import org.testar.visualvalidation.ocr.OcrResultCallback; + +import java.awt.image.BufferedImage; +import java.util.ArrayList; + +public class DummyOcrEngine implements OcrEngineInterface { + @Override + public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { + // Immediately trigger the callback. + callback.reportResult(new ArrayList<>()); + } + + @Override + public void Destroy() { + + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java new file mode 100644 index 000000000..974febcc2 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractOcrEngine.java @@ -0,0 +1,235 @@ +package org.testar.visualvalidation.ocr.tesseract; + +import org.apache.logging.log4j.Level; +import org.bytedeco.javacv.Java2DFrameConverter; +import org.bytedeco.javacv.LeptonicaFrameConverter; +import org.bytedeco.tesseract.ETEXT_DESC; +import org.bytedeco.tesseract.ResultIterator; +import org.bytedeco.tesseract.TessBaseAPI; +import org.bytedeco.tesseract.global.tesseract; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.testar.Logger; +import org.testar.extendedsettings.ExtendedSettingsFactory; +import org.testar.visualvalidation.ocr.OcrEngineInterface; +import org.testar.visualvalidation.ocr.OcrResultCallback; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.awt.image.DataBuffer; +import java.awt.image.DataBufferByte; +import java.awt.image.DataBufferInt; +import java.awt.image.DataBufferShort; +import java.awt.image.DataBufferUShort; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static java.awt.image.BufferedImage.TYPE_4BYTE_ABGR; +import static java.awt.image.BufferedImage.TYPE_INT_ARGB; +import static org.bytedeco.leptonica.global.lept.pixRead; +import static org.bytedeco.tesseract.global.tesseract.PSM_AUTO_OSD; + +/** + * OCR engine implementation dedicated for the Tesseract engine. + * Analyzes the given image via {@link #AnalyzeImage} in a separate thread. + * Once the engine has finished the caller shall be informed via the given callback. + *

+ * The current implementation can only process one image at the time. This is enforced by the synchronized statements in + * {@link #AnalyzeImage} where we set the buffer which is used by the engine and the actual execution of the engine in + * the {@link #run}. + */ +public class TesseractOcrEngine extends Thread implements OcrEngineInterface { + private static final String TAG = "Tesseract"; + private final TessBaseAPI _engine; + private final Object _scanSync = new Object(); + private final int _imageResolution; + private final boolean _loggingEnabled; + private final boolean _saveImageBufferToDisk; + AtomicBoolean running = new AtomicBoolean(true); + private BufferedImage _image = null; + private OcrResultCallback _callback = null; + + public TesseractOcrEngine() { + _engine = new TessBaseAPI(); + TesseractSettings config = ExtendedSettingsFactory.createTesseractSetting(); + _imageResolution = config.imageResolution; + _loggingEnabled = config.loggingEnabled; + _saveImageBufferToDisk = config.saveImageBufferToDisk; + + if (_engine.Init(config.dataPath, config.language) != 0) { + Logger.log(Level.ERROR, TAG, "Could not initialize tesseract."); + } + + if (_loggingEnabled) { + Logger.log(Level.INFO, TAG, "Tesseract engine created; Language:{} Data path:{}", + config.language, config.dataPath); + } + + setName(TAG); + start(); + } + + @Override + public void AnalyzeImage(BufferedImage image, OcrResultCallback callback) { + synchronized (_scanSync) { + _image = image; + _callback = callback; + if (_loggingEnabled) { + Logger.log(Level.TRACE, TAG, "Queue new image scan."); + } + _scanSync.notifyAll(); + } + } + + @Override + public void run() { + while (running.get()) { + synchronized (_scanSync) { + try { + // Wait until we need to inspect a new image. + _scanSync.wait(); + if (!running.get()) { + break; + } + recognizeText(); + + } catch (InterruptedException e) { + Logger.log(Level.ERROR, TAG, "Wait interrupted"); + e.printStackTrace(); + } + } + } + } + + private void recognizeText() { + if (_image == null || _callback == null) { + Logger.log(Level.ERROR, TAG, "Should not try to detect text on empty image/callback"); + return; + } + + List recognizedWords = new ArrayList<>(); + + _engine.SetPageSegMode(PSM_AUTO_OSD); + + loadImageIntoEngine(_image); + + if (_engine.Recognize(new ETEXT_DESC()) != 0) { + Logger.log(Level.ERROR, TAG, "Could not process image."); + } else { + try (ResultIterator recognizedElement = _engine.GetIterator()) { + int level = tesseract.RIL_WORD; + do { + recognizedWords.add(TesseractResult.Extract(recognizedElement, level)); + } while (recognizedElement.Next(level)); + } + } + _engine.Clear(); + + // Filter out the empty items and notify the callback with the discovered words. + _callback.reportResult(recognizedWords.stream() + .filter(recognizedElement -> !recognizedElement._text.isEmpty()) + .collect(Collectors.toList()) + ); + } + + private void loadImageIntoEngine(@NonNull BufferedImage image) { + // Ideally we use the raw data but unfortunately the webdriver screenshots have a different format which doesn't + // work with the current conversion for loading a raw buffer. + switch (image.getType()) { + case TYPE_INT_ARGB: + // Works for desktop protocol. + loadImageAsRawData(image); + break; + case TYPE_4BYTE_ABGR: + // Works for webdriver protocol. + loadImageAsPix(image); + break; + default: + throw new IllegalArgumentException("Loading OCR image not supported for image type:" + image.getType()); + } + } + + private void loadImageAsRawData(@NonNull BufferedImage image) { + DataBuffer dataBuffer = image.getData().getDataBuffer(); + + ByteBuffer byteBuffer; + if (dataBuffer instanceof DataBufferByte) { + byte[] pixelData = ((DataBufferByte) dataBuffer).getData(); + byteBuffer = ByteBuffer.wrap(pixelData); + } else if (dataBuffer instanceof DataBufferUShort) { + short[] pixelData = ((DataBufferUShort) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 2); + byteBuffer.asShortBuffer().put(ShortBuffer.wrap(pixelData)); + } else if (dataBuffer instanceof DataBufferShort) { + short[] pixelData = ((DataBufferShort) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 2); + byteBuffer.asShortBuffer().put(ShortBuffer.wrap(pixelData)); + } else if (dataBuffer instanceof DataBufferInt) { + int[] pixelData = ((DataBufferInt) dataBuffer).getData(); + byteBuffer = ByteBuffer.allocate(pixelData.length * 4); + byteBuffer.asIntBuffer().put(IntBuffer.wrap(pixelData)); + } else { + throw new IllegalArgumentException("Not implemented for data buffer type: " + dataBuffer.getClass()); + } + + // When applicable, alpha is included. + int bytes_per_pixel = image.getColorModel().getNumComponents(); + int bytes_per_line = bytes_per_pixel * image.getWidth(); + + _engine.SetImage(byteBuffer, image.getWidth(), image.getHeight(), bytes_per_pixel, bytes_per_line); + _engine.SetSourceResolution(_imageResolution); + } + + private void loadImageAsPix(@NonNull BufferedImage image) { + // Works for web driver + if (_saveImageBufferToDisk) { + try { + File outputFile = File.createTempFile("testar", ".png"); + outputFile.deleteOnExit(); + ImageIO.write(_image, "png", outputFile); + _engine.SetImage(pixRead(outputFile.getAbsolutePath())); + _engine.SetSourceResolution(_imageResolution); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + try (Java2DFrameConverter converter = new Java2DFrameConverter(); + LeptonicaFrameConverter converter2 = new LeptonicaFrameConverter()) { + // Convert the buffered image into a PIX. + _engine.SetImage(converter2.convert(converter.convert(image))); + } catch (Exception e) { + Logger.log(Level.ERROR, TAG, "Failed to convert buffered image into PIX"); + } + } + } + + @Override + public void Destroy() { + stopAndJoinThread(); + + _engine.End(); + if (_loggingEnabled) { + Logger.log(Level.DEBUG, TAG, "Engine destroyed."); + } + } + + private void stopAndJoinThread() { + synchronized (_scanSync) { + running.set(false); + _scanSync.notifyAll(); + } + + try { + join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractResult.java b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractResult.java new file mode 100644 index 000000000..a23763b52 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractResult.java @@ -0,0 +1,61 @@ +package org.testar.visualvalidation.ocr.tesseract; + +import org.apache.logging.log4j.Level; +import org.bytedeco.javacpp.BytePointer; +import org.bytedeco.javacpp.IntPointer; +import org.bytedeco.tesseract.ResultIterator; +import org.testar.Logger; +import org.testar.visualvalidation.Location; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.function.Supplier; + +/** + * Convert a tesseract result into a {@link RecognizedElement} + */ +public class TesseractResult { + + /** + * Convert a {@link TesseractResult} into a {@link RecognizedElement} + * + * @param recognizedElement The recognized element. + * @param granularity The granularity of recognized element, range from a block till individual character. + * See tesseract::PageIteratorLevel for more information. + * @return An {@link RecognizedElement}. + */ + static RecognizedElement Extract(ResultIterator recognizedElement, int granularity) { + Supplier intPointerSupplier = () -> new IntPointer(new int[1]); + + BytePointer ocrResult = recognizedElement.GetUTF8Text(granularity); + if (ocrResult == null) { + Logger.log(Level.ERROR, "OCR", "Results is null"); + return new RecognizedElement(new Location(), 0, ""); + } + String recognizedText = ocrResult.getString().trim(); + + float confidence = recognizedElement.Confidence(granularity); + IntPointer left = intPointerSupplier.get(); + IntPointer top = intPointerSupplier.get(); + IntPointer right = intPointerSupplier.get(); + IntPointer bottom = intPointerSupplier.get(); + boolean foundRectangle = recognizedElement.BoundingBox(granularity, left, top, right, bottom); + + if (!foundRectangle) { + throw new IllegalArgumentException("Could not find any rectangle for this element"); + } + + // Upper left coordinate = 0,0 + int width = right.get() - left.get(); + int height = bottom.get() - top.get(); + Location location = new Location(left.get(), top.get(), width, height); + RecognizedElement result = new RecognizedElement(location, confidence, recognizedText); + + left.deallocate(); + top.deallocate(); + right.deallocate(); + bottom.deallocate(); + ocrResult.deallocate(); + + return result; + } +} diff --git a/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractSettings.java b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractSettings.java new file mode 100644 index 000000000..8e22d2234 --- /dev/null +++ b/testar/src/org/testar/visualvalidation/ocr/tesseract/TesseractSettings.java @@ -0,0 +1,40 @@ +package org.testar.visualvalidation.ocr.tesseract; + +import org.testar.extendedsettings.ExtendedSettingBase; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class TesseractSettings extends ExtendedSettingBase { + public String dataPath; + public String language; + public int imageResolution; + public boolean loggingEnabled; + public boolean saveImageBufferToDisk; + + public static TesseractSettings CreateDefault() { + TesseractSettings instance = new TesseractSettings(); + instance.dataPath = System.getenv("LOCALAPPDATA") + "\\Tesseract-OCR\\tessdata"; + instance.language = "eng"; + instance.imageResolution = 144; + instance.loggingEnabled = false; + instance.saveImageBufferToDisk = true; + return instance; + } + + @Override + public int compareTo(TesseractSettings other) { + int result = -1; + if (language.contentEquals(other.language) + && (dataPath.contentEquals(other.dataPath)) + && (imageResolution == other.imageResolution) + && (loggingEnabled == other.loggingEnabled) + && (saveImageBufferToDisk == other.saveImageBufferToDisk)) { + result = 0; + } + return result; + } +} diff --git a/testar/test/org/testar/settings/XmlFile.java b/testar/test/org/testar/settings/XmlFile.java index 9944f7cbf..b1b9b46d9 100644 --- a/testar/test/org/testar/settings/XmlFile.java +++ b/testar/test/org/testar/settings/XmlFile.java @@ -99,4 +99,4 @@ private static void CreateFile(final String absolutePath, final String content) File testFile = new File(absolutePath); assertTrue(testFile.exists()); } -} \ No newline at end of file +} diff --git a/testar/test/org/testar/visualvalidation/matcher/ContentMatcherTest.java b/testar/test/org/testar/visualvalidation/matcher/ContentMatcherTest.java new file mode 100644 index 000000000..64a02dd96 --- /dev/null +++ b/testar/test/org/testar/visualvalidation/matcher/ContentMatcherTest.java @@ -0,0 +1,158 @@ +package org.testar.visualvalidation.matcher; + +import org.junit.Test; +import org.testar.visualvalidation.Location; +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class ContentMatcherTest { + + private static final RecognizedElement firstLine = new RecognizedElement(new Location(0, 0, 20, 10), 100, "1"); + private static final RecognizedElement secondLine = new RecognizedElement(new Location(0, 10, 20, 10), 100, "2W"); + private static final RecognizedElement thirdLineFirst = new RecognizedElement(new Location(0, 20, 20, 10), 100, "3 "); + private static final RecognizedElement thirdLineSecond = new RecognizedElement(new Location(10, 25, 20, 5), 100, "?"); + private static final ExpectedElement expectedElement = new ExpectedElement(new Location(0, 0, 20, 30), "1K\n\r2w\n\r3"); + + private static final MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + + private LocationMatch prepareExpectedTextWith3Lines() { + LocationMatch locationMatch = new LocationMatch(expectedElement, 0); + // Shuffled input so we can test the algorithm. + locationMatch.addRecognizedElement(firstLine); + locationMatch.addRecognizedElement(thirdLineFirst); + locationMatch.addRecognizedElement(secondLine); + locationMatch.addRecognizedElement(thirdLineSecond); + + return locationMatch; + } + + @Test + public void match() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we run the algorithm. + ContentMatchResult result = ContentMatcher.Match(input, config); + + // THEN the result must be: + assertEquals(9, result.totalExpected); + assertEquals(8, result.totalMatched); + + assertSame('1', result.expectedResult.expectedText.get(0).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(0).result); + assertSame('1', result.expectedResult.expectedText.get(0).character.match.character); + + assertSame('K', result.expectedResult.expectedText.get(1).character.character); + assertEquals(CharacterMatchResult.NO_MATCH, result.expectedResult.expectedText.get(1).result); + assertNull(result.expectedResult.expectedText.get(1).character.match); + + assertSame('\n', result.expectedResult.expectedText.get(2).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(2).result); + assertNull(result.expectedResult.expectedText.get(1).character.match); + + assertSame('\r', result.expectedResult.expectedText.get(3).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(3).result); + assertNull(result.expectedResult.expectedText.get(3).character.match); + + assertSame('2', result.expectedResult.expectedText.get(4).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(4).result); + assertSame('2', result.expectedResult.expectedText.get(4).character.match.character); + + assertSame('w', result.expectedResult.expectedText.get(5).character.character); + assertEquals(CharacterMatchResult.CASE_MISMATCH, result.expectedResult.expectedText.get(5).result); + assertSame('W', result.expectedResult.expectedText.get(5).character.match.character); + + assertSame('\n', result.expectedResult.expectedText.get(6).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(6).result); + assertNull(result.expectedResult.expectedText.get(6).character.match); + + assertSame('\r', result.expectedResult.expectedText.get(7).character.character); + assertEquals(CharacterMatchResult.WHITESPACE_CORRECTED, result.expectedResult.expectedText.get(7).result); + assertNull(result.expectedResult.expectedText.get(7).character.match); + + assertSame('3', result.expectedResult.expectedText.get(8).character.character); + assertEquals(CharacterMatchResult.MATCHED, result.expectedResult.expectedText.get(8).result); + assertSame('3', result.expectedResult.expectedText.get(8).character.match.character); + + assertSame('?', result.recognizedResult.recognized.get(5).character); + assertNull(result.recognizedResult.recognized.get(5).match); + } + + @Test + public void sortRecognizedElements() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we sort the recognized elements. + List result = ContentMatcher.sortRecognizedElements(input, config); + + // THEN the recognized elements should be sorted correctly. + assertEquals(Arrays.asList(firstLine, secondLine, thirdLineFirst, thirdLineSecond), result); + } + + @Test + public void sortRecognizedElementsPerTextLine() { + // GIVEN we have an expected text containing three text lines. + LocationMatch input = prepareExpectedTextWith3Lines(); + + // WHEN we sort the recognized elements only on their y coordinate. + Map> result = ContentMatcher.sortRecognizedElementsPerTextLine(input, config); + + // THEN the recognized elements should be sorted correctly. + Map> expectedResult = Stream.of( + new AbstractMap.SimpleEntry<>(5, Collections.singletonList(firstLine)), + new AbstractMap.SimpleEntry<>(15, Collections.singletonList(secondLine)), + new AbstractMap.SimpleEntry<>(27, Arrays.asList(thirdLineFirst, thirdLineSecond))) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + expectedResult.keySet().forEach(expectedKey -> + { + assertEquals(expectedResult.get(expectedKey).size(), result.get(expectedKey).size()); + expectedResult.get(expectedKey).forEach(it -> assertTrue(result.get(expectedKey).contains(it))); + } + ); + } + + @Test + public void correctAllNoneMatchedWhitespaces() { + // GIVEN a prepared result containing whitespace characters which are not marked as matched. + ExpectedTextMatchResult input = new ExpectedTextMatchResult(" T\te\fs\nt\r.\u000B?"); + Arrays.asList(1, 3, 5, 7, 9, 11).forEach(it -> + input.expectedText.get(it).result = CharacterMatchResult.MATCHED + ); + + // WHEN the correction mechanism is applied + ContentMatcher.correctWhitespaces(input); + + // THEN all the whitespaces should be corrected. + for (int i = 0; i < input.expectedText.size(); i++) { + assertSame(i % 2 == 0 ? CharacterMatchResult.WHITESPACE_CORRECTED : CharacterMatchResult.MATCHED, + input.expectedText.get(i).result); + } + } + + @Test + public void notCorrectMatchedWhitespaces() { + // GIVEN a prepared result containing whitespace characters which are matched. + ExpectedTextMatchResult input = new ExpectedTextMatchResult(" T"); + input.expectedText.forEach(it -> it.result = CharacterMatchResult.MATCHED); + + // WHEN the correction mechanism is applied + ContentMatcher.correctWhitespaces(input); + + // THEN all the whitespaces should be corrected. + input.expectedText.forEach(it -> assertSame(CharacterMatchResult.MATCHED, it.result)); + } +} diff --git a/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java b/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java new file mode 100644 index 000000000..a1305e79f --- /dev/null +++ b/testar/test/org/testar/visualvalidation/matcher/LocationMatcherTest.java @@ -0,0 +1,133 @@ +package org.testar.visualvalidation.matcher; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.testar.visualvalidation.Location; +import org.testar.visualvalidation.extractor.ExpectedElement; +import org.testar.visualvalidation.ocr.RecognizedElement; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(MockitoJUnitRunner.class) +public class LocationMatcherTest { + + private static final RecognizedElement recognizedElementOne = + new RecognizedElement(new Location(0, 15, 36, 1), 0, "r"); + private static final RecognizedElement recognizedElementTwo = + new RecognizedElement(new Location(40, 16, 26, 45), 23, "df"); + private static final RecognizedElement recognizedElementThree = + new RecognizedElement(new Location(5, 10, 26, 45), 23, "aa"); + private static final RecognizedElement recognizedElementFour = + new RecognizedElement(new Location(12, 16, 10, 20), 23, "Open"); + private static final ExpectedElement expectedElement = + new ExpectedElement(new Location(10, 15, 15, 23), "open"); + + @Test + public void successfulMatch() { + // GIVEN we have one expected text element with a matching(case corrected for first character) + // recognized element plus three additional garbage elements. + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + List recognizedElements = Arrays.asList( + recognizedElementOne, recognizedElementTwo, + recognizedElementThree, recognizedElementFour); + + List expectedElements = Collections.singletonList(expectedElement); + + // WHEN we run the match algorithm + MatcherResult result = sut.Match(recognizedElements, expectedElements); + + // THEN The match result should be: + // We expected that one recognized elements isn't matched based on the location. + assertEquals(1, result.getNoLocationMatches().size()); + assertTrue(result.getNoLocationMatches().contains(recognizedElementTwo)); + + // We have 3 recognized elements that intersect with the expected text. + assertEquals(1, result.getLocationMatches().size()); + LocationMatch locationMatch = result.getLocationMatches().iterator().next(); + assertEquals(3, locationMatch.recognizedElements.size()); + + // Validate that all expected characters have been matched. + assertEquals(1, result.getResult().size()); + assertEquals(4, result.getResult().iterator().next().totalMatched); + } + + @Test + public void setUnmatchedElements() { + // GIVEN we have a matched only the first recognized element. + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + MatcherResult matcherResult = Mockito.mock(MatcherResult.class); + LocationMatch locationMatchMock = Mockito.mock(LocationMatch.class); + locationMatchMock.recognizedElements = new HashSet<>(Collections.singletonList(recognizedElementOne)); + Mockito.when(matcherResult.getLocationMatches()).thenReturn(new HashSet<>(Collections.singletonList(locationMatchMock))); + + ArrayList recognizedElements = new ArrayList<>(Arrays.asList( + recognizedElementOne, + recognizedElementTwo, + recognizedElementThree + )); + + // WHEN we collect the unmatched results. + sut.setUnmatchedElements(matcherResult, recognizedElements); + + // THEN the matcher result should contain the two remaining recognized elements. + ArgumentCaptor arg = ArgumentCaptor.forClass(RecognizedElement.class); + verify(matcherResult, times(2)).addNoLocationMatch(arg.capture()); + assertEquals(2, recognizedElements.size()); + assertEquals(arg.getAllValues().get(0), recognizedElementTwo); + assertEquals(arg.getAllValues().get(1), recognizedElementThree); + } + + @Test + public void getIntersectedElements() { + // GIVEN two recognized elements inside and one outside the surface area of the expected text. + List recognizedElements = Arrays.asList( + recognizedElementOne, + recognizedElementTwo, + recognizedElementThree + ); + + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + // WHEN we try find intersecting elements. + LocationMatch result = sut.getIntersectedElements(recognizedElements, expectedElement); + + // THEN the two recognized elements inside the area mus tbe listed in the outcome. + assertNotNull(result); + assertEquals(expectedElement._location, result.location.location); + assertEquals(new HashSet<>(Arrays.asList(recognizedElementOne, recognizedElementThree)), result.recognizedElements); + } + + @Test + public void getIntersectedElementsReturnsNull() { + // GIVEN recognized elements outside the surface area of the expected text. + List recognizedElements = Collections.singletonList(recognizedElementTwo); + + MatcherConfiguration config = MatcherConfiguration.CreateDefault(); + LocationMatcher sut = new LocationMatcher(config); + + // WHEN we try find intersecting elements. + LocationMatch result = sut.getIntersectedElements(recognizedElements, expectedElement); + + // THEN none are expected. + assertNull(result); + } +} diff --git a/webdriver/src/org/testar/monkey/alayer/webdriver/WdProtocolUtil.java b/webdriver/src/org/testar/monkey/alayer/webdriver/WdProtocolUtil.java index 5c60a00ca..565dff336 100644 --- a/webdriver/src/org/testar/monkey/alayer/webdriver/WdProtocolUtil.java +++ b/webdriver/src/org/testar/monkey/alayer/webdriver/WdProtocolUtil.java @@ -54,7 +54,7 @@ public static String getStateshot(State state) { return ScreenshotSerialiser.saveStateshot(state.get(Tags.ConcreteIDCustom), screenshot); } - public static String getActionshot(State state, Action action) { + public static AWTCanvas getActionshot(State state, Action action) { List targets = action.get(Tags.Targets, null); if (targets == null) { return null; @@ -81,8 +81,7 @@ public static String getActionshot(State state, Action action) { Rect rect = Rect.from( actionArea.x, actionArea.y, actionArea.width + 1, actionArea.height + 1); - AWTCanvas scrshot = WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); - return ScreenshotSerialiser.saveActionshot(state.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), action.get(Tags.ConcreteIDCustom, "NoConcreteIdAvailable"), scrshot); + return WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); } public static AWTCanvas getStateshotBinary(State state) { @@ -91,9 +90,8 @@ public static AWTCanvas getStateshotBinary(State state) { && state.get(WdTags.WebHorizontallyScrollable, null) == null) { //Get a screenshot of all the screen, because SUT ended and we can't obtain the size Rectangle screenRect = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); - AWTCanvas scrshot = AWTCanvas.fromScreenshot(Rect.from(screenRect.getX(), screenRect.getY(), + return AWTCanvas.fromScreenshot(Rect.from(screenRect.getX(), screenRect.getY(), screenRect.getWidth(), screenRect.getHeight()), state.get(Tags.HWND, (long)0), AWTCanvas.StorageFormat.PNG, 1); - return scrshot; } double width = CanvasDimensions.getCanvasWidth() + ( @@ -101,7 +99,6 @@ public static AWTCanvas getStateshotBinary(State state) { double height = CanvasDimensions.getCanvasHeight() + ( state.get(WdTags.WebHorizontallyScrollable) ? scrollThick : 0); Rect rect = Rect.from(0, 0, width, height); - AWTCanvas screenshot = WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); - return screenshot; + return WdScreenshot.fromScreenshot(rect, state.get(Tags.HWND, (long)0)); } -} \ No newline at end of file +} diff --git a/webdriver/src/org/testar/monkey/alayer/webdriver/WdScreenshot.java b/webdriver/src/org/testar/monkey/alayer/webdriver/WdScreenshot.java index e4d30be6a..7d630df13 100644 --- a/webdriver/src/org/testar/monkey/alayer/webdriver/WdScreenshot.java +++ b/webdriver/src/org/testar/monkey/alayer/webdriver/WdScreenshot.java @@ -39,7 +39,7 @@ import javax.imageio.ImageIO; import java.awt.image.BufferedImage; -import java.io.File; +import java.io.ByteArrayInputStream; /** * Extend AWTCanvas to get the screenshot from WebDriver @@ -52,21 +52,22 @@ private WdScreenshot() { } public static WdScreenshot fromScreenshot(Rect r, long windowHandle) - throws StateBuildException { + throws StateBuildException { WdScreenshot wdScreenshot = new WdScreenshot(); RemoteWebDriver webDriver = WdDriver.getRemoteWebDriver(); try { - File screenshot = webDriver.getScreenshotAs(OutputType.FILE); - BufferedImage fullImg = ImageIO.read(screenshot); - double displayScale = Environment.getInstance().getDisplayScale(windowHandle); - int x = (int) Math.max(0, r.x() * displayScale); - int y = (int) Math.max(0, r.y() * displayScale); - int width = (int) Math.min(fullImg.getWidth(), r.width() * displayScale); - int height = (int) Math.min(fullImg.getHeight(), r.height() * displayScale); - wdScreenshot.img = fullImg.getSubimage(x, y, width, height); - } - catch (Exception ignored) { + byte[] image = webDriver.getScreenshotAs(OutputType.BYTES); + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(image)) { + BufferedImage fullImg = ImageIO.read(inputStream); + double displayScale = Environment.getInstance().getDisplayScale(windowHandle); + int x = (int) Math.max(0, r.x() * displayScale); + int y = (int) Math.max(0, r.y() * displayScale); + int width = (int) Math.min(fullImg.getWidth(), r.width() * displayScale); + int height = (int) Math.min(fullImg.getHeight(), r.height() * displayScale); + wdScreenshot.img = fullImg.getSubimage(x, y, width, height); + } + } catch (Exception ignored) { } return wdScreenshot; diff --git a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java index 3d1dba92a..637b2e5fc 100644 --- a/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java +++ b/windows/src/org/testar/monkey/alayer/windows/StateFetcher.java @@ -104,7 +104,7 @@ public UIAState call() throws Exception { UIAState root = createWidgetTree(uiaRoot); root.set(Tags.Role, Roles.Process); root.set(Tags.NotResponding, false); - + root.set(Tags.HWND, uiaRoot.children.get(0).get(Tags.HWND, 0L)); for (Widget w : root) w.set(Tags.Path,Util.indexString(w)); if (system != null && (root == null || root.childCount() == 0) && system.getNativeAutomationCache() != null) @@ -327,7 +327,7 @@ private UIAElement uiaDescend(long hwnd, long uiaCachePointer, UIAElement parent long uiaWindowPointer = Windows.IUIAutomationElement_GetPattern(uiaCachePointer, Windows.UIA_WindowPatternId, true); if(uiaWindowPointer != 0){ uiaElement.wndInteractionState = Windows.IUIAutomationWindowPattern_get_WindowInteractionState(uiaWindowPointer, true); - uiaElement.blocked = (uiaElement.wndInteractionState != Windows.WindowInteractionState_ReadyForUserInteraction); + uiaElement.blocked = isElementBlocked(uiaElement); uiaElement.isTopmostWnd = Windows.IUIAutomationWindowPattern_get_IsTopmost(uiaWindowPointer, true); uiaElement.isModal = Windows.IUIAutomationWindowPattern_get_IsModal(uiaWindowPointer, true); @@ -508,7 +508,18 @@ private UIAElement uiaDescend(long hwnd, long uiaCachePointer, UIAElement parent return modalElement; } - + + private boolean isElementBlocked(UIAElement uiaElement) { + // Qt applications are always started in the running state. + // Without this dedicated check TESTAR can't find any actions for the Qt application. + if (Objects.equals(uiaElement.frameworkId, "Qt")) { + return !(uiaElement.wndInteractionState == Windows.WindowInteractionState_ReadyForUserInteraction || + uiaElement.wndInteractionState == Windows.WindowInteractionState_Running); + } else { + return (uiaElement.wndInteractionState != Windows.WindowInteractionState_ReadyForUserInteraction); + } + } + // (through AccessBridge) private UIAElement abDescend(long hwnd, UIAElement parent, long vmid, long ac){ UIAElement modalElement = null; diff --git a/windows/src/org/testar/monkey/alayer/windows/UIAState.java b/windows/src/org/testar/monkey/alayer/windows/UIAState.java index 9bf7f4c11..99582fc9d 100644 --- a/windows/src/org/testar/monkey/alayer/windows/UIAState.java +++ b/windows/src/org/testar/monkey/alayer/windows/UIAState.java @@ -254,8 +254,8 @@ Iterable> tags(final UIAWidget widget){ // compile a query set final Set> queryTags = new HashSet>(); queryTags.addAll(tags.keySet()); // the tags that have been set on this widget (state is also a widget) - queryTags.addAll(Tags.tagSet()); // the tags defined in org.fruit.alayer.Tags - queryTags.addAll(UIATags.tagSet()); // the tags defined in org.fruit.alayer.windows.UIATags + queryTags.addAll(Tags.tagSet()); // the tags defined in org.testar.monkey.alayer.Tags + queryTags.addAll(UIATags.tagSet()); // the tags defined in org.testar.monkey.alayer.windows.UIATags Iterable> returnTags = new Iterable>(){ public Iterator> iterator() {