Added a counter

Testing/LoggingFormatter.java

@@ -0,0 +1,28 @@
+package testing;
+
+import java.util.logging.LogRecord;
+import java.util.logging.SimpleFormatter;
+
+public class LoggingFormatter extends SimpleFormatter
+{
+
+	@Override
+	public synchronized String format(LogRecord record)
+	{
+		StringBuilder message = new StringBuilder();
+
+
+		message.append(record.getLevel().toString());
+		message.append(": ");
+		message.append(record.getMessage());
+		message.append("\n");
+		Throwable thrown = record.getThrown();
+		if (thrown != null)
+		{
+			message.append(record.getThrown());
+		}
+
+		return message.toString();
+	}
+
+}

Testing/Main.java

@@ -1,8 +1,98 @@
-package Testing;
+package testing;
 
-public class Main {
-    public static void main(String[] args){
-        Test t = new Test("~FOLDERNAME~", () -> ~PACKAGENAME~.Program.main(new String[]{}));
-        t.testAll();
-    }
+import java.io.File;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.Arrays;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class Main
+{
+
+	private final static Logger LOGGER = Logger.getLogger(Test.class.getName());
+	private static Level streamLogLevel = Level.FINE;
+	private static Level fileLogLevel = Level.ALL;
+
+
+	static
+	{
+		LOGGER.setUseParentHandlers(false);
+		java.util.logging.StreamHandler shandler = new java.util.logging.StreamHandler(
+				StreamHandler.out, new LoggingFormatter());
+		LOGGER.addHandler(shandler);
+	}
+
+	public static void main(String[] args)
+	{
+
+		LOGGER.log(Level.INFO, "Parameters: " + Arrays.toString(args));
+		if (args.length != 3)
+		{
+			LOGGER.log(
+					Level.SEVERE,
+					"Invalid Parameters. Syntax: testGAD.jar testFolder programPath packageName.className\n"
+							+ "The testFolder and programPath needs to be a directory. className must be the .class file which"
+							+ " contains the main method of the program to test.\n"
+							+ "Example: java -jar testGAD.jar ./Test ./dhashing dhashing.Program");
+		}
+		else
+		{
+			try
+			{
+				File testFolder = new File(args[0]);
+				File programPath = new File(args[1]);	
+				String className = args[2].replace(".class", "");
+
+				LOGGER.log(Level.INFO,"programPath contains: " +  Arrays.toString(programPath.list()));
+				System.out.println(testFolder.isDirectory() +" "+ programPath.isDirectory() );
+
+				URL url = programPath.toURI().toURL();
+				URL[] urls = new URL[] { url };
+				ClassLoader cl = new URLClassLoader(urls);
+
+
+				Class<?> mainClass = Class.forName(className, true, cl);
+				Method main = mainClass.getMethod("main", String[].class);
+				Test test = new Test(testFolder.getAbsolutePath(), main);
+				test.testAll();
+
+			}
+			catch (ClassNotFoundException e)
+			{
+				LOGGER.log(Level.SEVERE, "Class not found", e);
+			}
+			catch (MalformedURLException e)
+			{
+				LOGGER.log(Level.SEVERE,
+						"Invalid path and/or name for program", e);
+			}
+			catch (NoSuchMethodException e)
+			{
+				LOGGER.log(Level.SEVERE, "main method not found", e);
+			}
+			catch (SecurityException e)
+			{
+				LOGGER.log(Level.SEVERE, "access to main method denied", e);
+			}
+			finally
+			{
+				LOGGER.log(Level.INFO, "testGad terminated");
+			}
+		}
+
+	}
+
+	public static Level getStreamLogLevel()
+	{
+		return streamLogLevel;
+	}
+
+	public static Level getFileLogLevel()
+	{
+		return fileLogLevel;
+	}
+
 }

Testing/StreamHandler.java

@@ -1,17 +1,16 @@
-package Testing;
+package testing;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
-import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
 
 public class StreamHandler{
     private static final InputStream sIn = System.in;
-    public static final PrintStream out = System.out;
+    protected static final PrintStream out = System.out;
 
     public static ByteArrayOutputStream setOutputStream(int i){
 

Testing/Test.java

@@ -1,83 +1,168 @@
-package Testing;
+package testing;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
-import java.util.Arrays;
+import java.lang.reflect.Method;
+import java.util.HashSet;
 import java.util.Scanner;
+import java.util.Set;
+import java.util.logging.FileHandler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.stream.Stream;
 
+public class Test
+{
+
+	private String directory;
+	private Method main;
+	private static ByteArrayOutputStream generatedOutputStream = new ByteArrayOutputStream();
+	private static final Logger LOGGER = Logger.getLogger(Test.class.getName());
+	private static final Level failLogLevel = Level.FINE;
+	private static final Level succLogLevel = Level.FINER;
+
+	static {
+		LOGGER.setUseParentHandlers(false);
+		LOGGER.setLevel(Level.ALL);
+		java.util.logging.StreamHandler shandler= new java.util.logging.StreamHandler(StreamHandler.out, new LoggingFormatter());
+		shandler.setLevel(Main.getStreamLogLevel());
+		LOGGER.addHandler(shandler);
+		try
+		{
+			FileHandler fhandler = new FileHandler("./results.txt");
+			fhandler.setLevel(Main.getFileLogLevel());
+			LOGGER.addHandler(fhandler);
+		}
+		catch (Exception e)
+		{
+			LOGGER.log(Level.WARNING, "Logging to file failed", e);
+		}
+	}
+
+	public Test(String dir, Method main)
+	{
+
+		directory = dir;
+		this.main = main;
+	}
 
-public class Test {
-    private int testAmount;
-    private String directory;
-    private TestFunction testFunc;
-    private static ByteArrayOutputStream generatedOutputStream = new ByteArrayOutputStream();
-
-    public Test(String dir, TestFunction t) {
-        testAmount = new File(dir).list().length/2;
-        directory = dir;
-        testFunc = t;
-    }
-
-
-    public void testAll(){
-        String[] data = new File(directory).list();
-        data = Stream.of(data).filter(s -> s.contains(".in")).toArray(size -> new String[size]);
-        for(String e : data){
-            e = e.replace(".in", "");
-            System.out.println("------------------------\n"
-                             + "Case "+e+"\n"
-                             + "------------------------");
-            if(!testCase(e)) break;
-        }
-    }
-
-    public boolean testCase(String name){
-        String filenameIn  = directory + "/" + name + ".in";
-        String filenameOut = directory + "/" + name + ".out";
-        StreamHandler.setInputFileStream(filenameIn);
-        generatedOutputStream = new ByteArrayOutputStream();
-        StreamHandler.setOutputStream(generatedOutputStream);
-
-        testFunc.invoke();
-
-        StreamHandler.resetStreams();
-        return checkSolution(filenameOut);
-
-    }
-
-    private boolean checkSolution(String filenameOut){
-        //Trim is used to remove any whitespace surrounding the output
-        String correctOutput = getCorrectOutput(filenameOut).trim().replaceAll("(\\r|\\n|\\r\\n)+", "");
-        String generatedOutput = generatedOutputStream.toString().trim().replaceAll("(\\r|\\n|\\r\\n)+", "");
-        boolean check = generatedOutput.equals(correctOutput);
-
-        StreamHandler.out.println("Correct Output:   "+correctOutput);
-        StreamHandler.out.println("Generated Output: "+generatedOutput);
-        StreamHandler.out.println(check);
-
-        return check;
-    }
-
-    private String getCorrectOutput(String filename){
-        FileReader r = null;
-        try {
-            r = new FileReader(new File(filename));
-        } catch (FileNotFoundException e) {
-            System.out.println("File not found: "+filename);
-        }
-        Scanner s = new Scanner(r);
-        String correctOutput = "";
-        while(s.hasNext()) correctOutput += s.nextLine();
-        s.close();
-        return correctOutput;
-
-    }
+	public void testAll()
+	{
+
+
+		long start = System.currentTimeMillis();
+		String[] data = new File(directory).list();
+		data = Stream.of(data).filter(s -> s.contains(".in"))
+				.toArray(size -> new String[size]);
+		Set<String> badRuns = new HashSet<String>();
+		for (String caseName : data)
+		{
+			caseName = caseName.replace(".in", "");
+			if (!testCase(caseName))
+			{
+				logCase(failLogLevel, caseName);
+				badRuns.add(caseName);
+			}
+			else {
+				logCase(succLogLevel, caseName);
+			}
+		}
+
+		LOGGER.log(Level.INFO,
+				"Tests successful: " + (data.length - badRuns.size()) + "/"
+						+ data.length);
+		LOGGER.log(Level.INFO, "Failed Tests: " + badRuns.toString());
+		long totalTime = System.currentTimeMillis() - start;
+		LOGGER.log(
+				Level.INFO,
+				"Testing took " + totalTime + " ms or "
+						+ Math.round((double) totalTime / 1000) + " s");
+
+		LOGGER.exiting(Test.class.getName(), "testAll()");
+
+	}
+
+	public boolean testCase(String name)
+	{
+		String filenameIn = directory + "/" + name + ".in";
+		String filenameOut = directory + "/" + name + ".out";
+		StreamHandler.setInputFileStream(filenameIn);
+		generatedOutputStream = new ByteArrayOutputStream();
+		StreamHandler.setOutputStream(generatedOutputStream);
+
+		try
+		{
+			String[] args = new String[]{};
+			main.invoke(main.getDeclaringClass(), (Object)args);
+		}
+		catch (Exception e)
+		{
+			LOGGER.log(Level.SEVERE, "Exception thrown while invoking main method: ", e);
+		}
+
+		StreamHandler.resetStreams();
+		return checkSolution(filenameOut);
+
+	}
+
+	private boolean checkSolution(String filenameOut)
+	{
+		// Trim is used to remove any whitespace surrounding the output
+		String correctOutput = getCorrectOutput(filenameOut).trim().replaceAll(
+				"(\\r|\\n|\\r\\n)+", "");
+		String generatedOutput = generatedOutputStream.toString().trim()
+				.replaceAll("(\\r|\\n|\\r\\n)+", "");
+		boolean check = generatedOutput.equals(correctOutput);
+
+		if (check)
+		{
+			LOGGER.log(succLogLevel, "Correct Output:   " + correctOutput);
+			LOGGER.log(succLogLevel, "Generated Output: " + generatedOutput);
+			LOGGER.log(succLogLevel, "Successful");
+		}
+		else
+		{
+			LOGGER.log(failLogLevel, "Correct Output:   " + correctOutput);
+			LOGGER.log(failLogLevel, "Generated Output: " + generatedOutput);
+			LOGGER.log(failLogLevel, "Fail");
+		}
+
+		return check;
+	}
+
+	private String getCorrectOutput(String filename)
+	{
+		FileReader r = null;
+		try
+		{
+			r = new FileReader(new File(filename));
+		}
+		catch (FileNotFoundException e)
+		{
+			LOGGER.log(Level.SEVERE, "File not found: " + filename + "\t", e);
+		}
+		Scanner s = new Scanner(r);
+		StringBuilder correctOutput = new StringBuilder();
+		while (s.hasNext())
+		{
+			correctOutput.append(s.nextLine());
+		}
+		s.close();
+		return correctOutput.toString();
+
+	}
+
+	private void logCase(Level logLevel, String caseName) {
+
+		LOGGER.log(logLevel, "------------------------\n" + "Case " + caseName
+				+ "\n" + "------------------------");
+	}
 
 }
 
-interface TestFunction {
-    public void invoke();
+interface TestFunction
+{
+	public void invoke();
 }