Java automation

.gitignore

@@ -5,3 +5,4 @@ target/
 javaAnalysis.properties
 .idea/
 *.iml
+*.properties

README.md

@@ -4,7 +4,7 @@ This project aims at mining public software repositories on GitHub in order to m
 Properties file:
 The properties file provides a way to supply information for running the Java code without pushing private or unnecessary data to the repository.
 One needs to create a file called javaAnalysis.properties file in the same directory as the project (above src).
-The config.Config class communicates with this .properties file to supply information to the Java project.
+If information is left out but is necessary for running, it will ask for these on the command line.
 
 
 The structure of the .properties file is as follows (don't include the brackets):

pythonAnalysis.properties

@@ -0,0 +1,7 @@
+authToken =
+repoURLsPath=
+mongoUsername =
+mongoPassword =
+mongoUrl =
+mongoDatabase =
+mongoCollection =

src/FileParser.java

@@ -1,13 +1,4 @@
-import java.io.*;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Optional;
-
-import com.github.javaparser.JavaParser;
 import com.github.javaparser.ParseProblemException;
-import com.github.javaparser.ParserConfiguration;
 import com.github.javaparser.StaticJavaParser;
 import com.github.javaparser.ast.CompilationUnit;
 import com.github.javaparser.ast.ImportDeclaration;
@@ -19,6 +10,10 @@
 import config.Config;
 import org.json.JSONObject;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.*;
+
 public class FileParser {
 	// Parser objects -- for calling methods in other classes
 	private ParsingHelper ph = new ParsingHelper();
@@ -38,8 +33,7 @@ public class FileParser {
 	protected ArrayList<MethodWhiteSpaceResults> methodWPs = new ArrayList<>();
 	protected NameResults nr = new NameResults();
 
-	private static final Config config = Config.getInstance();
-	private static final String tempFilePath = config.getTempJavaFilePath();
+	private static final String tempFilePath = Config.getTempJavaFilePath();
 
 	private JSONify jsonify;
 

src/GetRepoNames.java

@@ -0,0 +1,67 @@
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import java.util.ArrayList;
+
+public class GetRepoNames {
+    public ArrayList<String> getReposByKeywords (int resultLimit, ArrayList<String> keywords) {
+        ArrayList results = new ArrayList();
+        GitHubFetcher fetcher = new GitHubFetcher();
+
+        try {
+            double originalLimit = new Double (resultLimit);
+
+            for (String word : keywords) {
+                //Mixes up the categories a bit among the keywords
+                int numRepos = (int) Math.ceil(originalLimit / keywords.size());
+                numRepos = (numRepos > resultLimit)? resultLimit : numRepos;
+                resultLimit -= numRepos;
+
+                if (numRepos == 0) {
+                    break;
+                }
+
+                JSONObject result = fetcher.makeGetRequest("https://api.github.com/search/repositories?q=" +
+                        word + "+language:java&sort=stars&order=desc&per_page=" + numRepos + "&page=1");
+                JSONArray jsonArray = result.getJSONArray("items");
+                for (int i = 0, size = jsonArray.length(); i < size; i++) {
+                    JSONObject objectInArray = jsonArray.getJSONObject(i);
+                    String r = "https://api.github.com/repos/" + objectInArray.getString("full_name");
+                    r += fetcher.getDefaultBranch(r);
+                    results.add(r);
+                }
+            }
+        } catch (CustomException e) {
+            e.printStackTrace();
+        } catch (JSONException e) {
+            e.printStackTrace();
+        }
+
+        return results;
+    }
+
+    public ArrayList<String> getReposByStars (int resultLimit) {
+        ArrayList results = new ArrayList();
+        GitHubFetcher fetcher = new GitHubFetcher();
+
+        try {
+            JSONObject result = fetcher.makeGetRequest("https://api.github.com/search/repositories?" +
+                    "q=language:java&sort=stars&order=desc&per_page=" + resultLimit + "&page=1");
+            JSONArray jsonArray = result.getJSONArray("items");
+            for (int i = 0, size = jsonArray.length(); i < size; i++) {
+                JSONObject objectInArray = jsonArray.getJSONObject(i);
+                String r = "https://api.github.com/repos/" + objectInArray.getString("full_name");
+                r += fetcher.getDefaultBranch(r);
+                results.add(r);
+            }
+        } catch (CustomException e) {
+            e.printStackTrace();
+        } catch (JSONException e) {
+            e.printStackTrace();
+        }
+
+        return results;
+    }
+}
+

src/GitHubFetcher.java

@@ -0,0 +1,82 @@
+import config.Config;
+import org.json.JSONObject;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.time.Instant;
+import java.util.concurrent.TimeUnit;
+
+public class GitHubFetcher {
+    public JSONObject makeGetRequest(String urlString) throws CustomException {
+        try {
+            String authToken = Config.getAuthToken();
+            URL url = new URL(urlString);
+            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+            conn.setRequestProperty("Authorization", "Bearer " + authToken);
+            conn.setRequestProperty("User-Agent", "code-style-mining");
+            conn.setRequestProperty("Content-Type","application/json");
+            conn.setDoOutput(true);
+            conn.setDoInput(true);
+            conn.setRequestMethod("GET");
+
+            if (conn.getResponseCode() == 403) {
+                reachedAPIRateLimit();
+                makeGetRequest(urlString);
+            }
+
+            BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+            String inputLine;
+            StringBuffer response = new StringBuffer();
+
+            while((inputLine = in.readLine()) != null) {
+                response.append(inputLine + "\n");
+            }
+            in.close();
+            return new JSONObject(response.toString());
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        throw new CustomException("Could not make get request.");
+    }
+
+    public void reachedAPIRateLimit() throws CustomException {
+        String urlString = "https://api.github.com/rate_limit";
+        try {
+            JSONObject respJSON = makeGetRequest(urlString);
+            int coreRemaining = respJSON.getJSONObject("resources").getJSONObject("core").getInt("remaining");
+            int searchRemaining = respJSON.getJSONObject("resources").getJSONObject("search").getInt("remaining");
+            if (coreRemaining == 0) {
+                int resetTime = respJSON.getJSONObject("resources").getJSONObject("core").getInt("reset");
+                int currentTime = (int) Instant.now().getEpochSecond();
+                int sleepTime = resetTime - currentTime;
+                if (sleepTime > 0) {
+                    sleepTime += (60 * 5);
+                    System.out.println(
+                        "reached API rate limit of 5000 per hour... sleeping for " + sleepTime +
+                        " seconds (" + (sleepTime/60) + " minutes)"
+                    );
+                    TimeUnit.SECONDS.sleep(sleepTime);
+                }
+            } else if (searchRemaining == 0) {
+                System.out.println("reached minute API rate limit of 30 per minute... sleeping for 60 seconds");
+                TimeUnit.SECONDS.sleep(60);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public String getDefaultBranch(String url) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("/branches/");
+        try {
+            JSONObject response = makeGetRequest(url);
+            sb.append(response.getString("default_branch"));
+        } catch(Exception e) {
+            e.printStackTrace();
+        }
+        return sb.toString();
+    }
+}

src/JSONify.java

@@ -7,7 +7,7 @@
 
 public class JSONify {
     private FileParser fp;
-    private String tempJSONFilePath = Config.getInstance().getTempJSONFilePath();
+    private String tempJSONFilePath = Config.getTempJSONFilePath();
 
     public JSONify(FileParser fp) {
         this.fp = fp;

src/MainParser.java

@@ -1,19 +1,159 @@
+import config.Config;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.FileBasedConfiguration;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
+import org.apache.commons.configuration2.builder.fluent.Parameters;
+import org.apache.commons.configuration2.ex.ConfigurationException;
 import org.json.JSONException;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+
 public class MainParser {
 //	public static final String gitURL = "https://github.com/janani-sridhar/CaesarCipher";
 //	public static final String directory = "/Users/jananisridhar/Desktop/CC";
 //	public static final String localFile = "./RandomFile.java";
 //
 //	public static FileParser fp;
 //	public static JSONifySummary summary;
-
+
+
+	public static void testTermination(String s) {
+		if(s.equals("q")) {
+			System.exit(0);
+		}
+	}
+
 	public static void main(String[] args) throws JSONException {
 		//summary = new JSONifySummary();
 		//fp = new FileParser(summary);
 		RepoTraversal traverser = new RepoTraversal();
+		Scanner reader = new Scanner(System.in);
+
+		Map<String, String> configMap = new HashMap<>();
+		configMap.put("authToken", null);
+		configMap.put("mongoUrl", null);
+		configMap.put("mongoUsername", null);
+		configMap.put("mongoPassword", null);
+		configMap.put("mongoDatabase", null);
+		configMap.put("mongoCollection", null);
+		configMap.put("tempJavaFilePath", null);
+		configMap.put("tempJSONFilePath", null);
+		configMap.put("repoURLsPath", null);
+
+		Parameters params = new Parameters();
+		FileBasedConfigurationBuilder<FileBasedConfiguration> builder =
+				new FileBasedConfigurationBuilder<FileBasedConfiguration>(PropertiesConfiguration.class)
+						.configure(params.properties().setFileName("javaAnalysis.properties"));
+		try {
+			Configuration config = builder.getConfiguration();
+			String temp;
+			//Properties not found will remain null
+			temp = config.getString("tempJavaFilePath");
+			configMap.put("tempJavaFilePath", (temp !=null)? temp.replaceAll("\"", "") : null);
+			temp = config.getString("tempJSONFilePath");
+			configMap.put("tempJSONFilePath", (temp !=null)? temp.replaceAll("\"", "") : null);
+			temp = config.getString("authToken");
+			configMap.put("authToken", (temp !=null)? temp.replaceAll("\"", "") : null);
+			temp = config.getString("repoURLsPath");
+			configMap.put("repoURLsPath", (temp !=null)? temp.replaceAll("\"", "") : null);
+
+			configMap.put("mongoUsername", config.getString("mongoUsername"));
+			configMap.put("mongoPassword", config.getString("mongoPassword"));
+			configMap.put("mongoUrl", config.getString("mongoUrl"));
+			configMap.put("mongoDatabase", config.getString("mongoDatabase"));
+			configMap.put("mongoCollection", config.getString("mongoCollection"));
+		}
+		catch(ConfigurationException e) {
+			System.out.println("No javaAnalysis.properties found...\n");
+			//e.printStackTrace();
+		}
+
+		//Ask for missing creds via the CL (exception for filename)
+		for (Map.Entry<String, String> entry : configMap.entrySet()) {
+			String key = entry.getKey();
+
+			if (key.equals("repoURLsPath")) {
+				continue;
+			}
+
+			while (configMap.get(key) == null || configMap.get(key).isEmpty()) {
+				System.out.print("Missing " + key + ". Enter one or [q] to quit: ");
+				String s = reader.nextLine().trim();
+				System.out.print("\n");
+				testTermination(s.toLowerCase());
+				configMap.put(key, s);
+			}
+		}
+
+		String inputType = "";
+		Integer limitRepos = null;
+		ArrayList<String> keywords = new ArrayList<>();
+		String optionType = "";
+		while (!inputType.equals("f") && !inputType.equals("g")) {
+			System.out.print("Do you want to get repo names from file [f] or generate them [g]. Enter one or [q] to quit: ");
+			inputType = reader.nextLine().trim();
+			System.out.print("\n");
+			testTermination(inputType.toLowerCase());
+		}
+
+		if (inputType.equals("f")) {
+			while (configMap.get("repoURLsPath") == null || configMap.get("repoURLsPath").isEmpty()) {
+				System.out.print("Missing repoURLsPath. Enter one or [q] to quit: ");
+				String s = reader.nextLine().trim();
+				System.out.print("\n");
+				testTermination(s.toLowerCase());
+				configMap.put("repoURLsPath", s);
+			}
+		} else {
+			while (!optionType.equals("s") && !optionType.equals("k")) {
+				System.out.print("Get repo names by top stars [s] or by keywords [k]. Enter one of these or [q] to quit: ");
+				optionType = reader.nextLine().trim();
+				System.out.print("\n");
+				testTermination(optionType.toLowerCase());
+			}
+
+			if (optionType.equals("k")) {
+				String t = "";
+				while (!t.equals("c")) {
+					System.out.print("Enter keyword or [c] to continue: ");
+					t = reader.nextLine().trim();
+					System.out.print("\n");
+					if (t.equals("c")) {
+						break;
+					} else if (!t.isEmpty()){
+						keywords.add(t);
+					}
+				}
+				System.out.println("Using keywords: " + keywords);
+			}
+
+			while (limitRepos == null) {
+				System.out.print("How many repos do you want to fetch (final results might not match this number)?" +
+						" Enter one or [q] to quit: ");
+				String s = reader.nextLine().trim();
+				System.out.print("\n");
+				testTermination(s.toLowerCase());
+				try {
+					if (Integer.parseInt(s) >= 0) {
+						limitRepos = Integer.parseInt(s);
+					}
+				} catch (NumberFormatException e) {
+					continue;
+				}
+			}
+		}
+
+		Config.init(configMap.get("authToken"), configMap.get("mongoUsername"), configMap.get("mongoPassword"),
+				configMap.get("mongoUrl"), configMap.get("mongoDatabase"), configMap.get("mongoCollection"),
+				configMap.get("tempJavaFilePath"), configMap.get("tempJSONFilePath"), configMap.get("repoURLsPath"));
 
-		traverser.findJavaFilesToParse();
+		//TODO might want to handle these options more dynamically.
+		// In some cases you dont need all of these put they are passed anyway
+		traverser.findJavaFilesToParse(inputType, optionType, limitRepos, keywords);
 	}
 
 	/*public static void getGitFiles(String url, String directory) {