From 43d7bb99ef2c3d7bb3bd08b96501163767cef372 Mon Sep 17 00:00:00 2001 From: Irineu Date: Tue, 7 Jan 2014 22:35:47 -0300 Subject: [PATCH 01/11] Merging SearchGitHub with fixes for quoted user names and missing issues metadata --- .../groundhog/search/SearchGitHubTest.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java index d9d640c..1d16a0a 100644 --- a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java +++ b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java @@ -7,6 +7,7 @@ import org.junit.Test; import br.ufpe.cin.groundhog.Commit; +import br.ufpe.cin.groundhog.Issue; import br.ufpe.cin.groundhog.Language; import br.ufpe.cin.groundhog.Project; import br.ufpe.cin.groundhog.Release; @@ -118,4 +119,43 @@ public void testGetAllProjectReleases() { Assert.fail(); } } + + @Test + public void testGetProjectIssues(){ + try { + User u = new User("spgroup"); + Project project = new Project(u, "groundhog"); + + List projectIssues = searchGitHub.getProjectIssues(project, 2); + + Assert.assertNotNull(projectIssues); + // The default number of items per page is 30 so for 2 pages we should have 60 items + // As of 01-Jan-2014 the number of issues for project spgroup/groundhog is 315 so it + // should be guaranteed to have 2 pages at least. + Assert.assertEquals(projectIssues.size(), 60); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } + + // Note that in the future this test may have to be reworked as the number of issues in the + // target project grows + @Test + public void testGetAllProjectIssues(){ + try { + User u = new User("spgroup"); + Project project = new Project(u, "groundhog"); + + List projectIssues = searchGitHub.getAllProjectIssues(project); + + Assert.assertNotNull(projectIssues); + // As of 01-Jan-2014 the number of issues for project spgroup/groundhog is 315 and + // this number can only grow + Assert.assertTrue(projectIssues.size() >= 315); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } } \ No newline at end of file From 1f46592160202f29404f18dc7ab2ed2855b45449 Mon Sep 17 00:00:00 2001 From: Irineu Date: Sun, 5 Jan 2014 11:52:11 -0300 Subject: [PATCH 02/11] Improving project issues search Reimplementing issues search using new Github API V3 Adding new issues search API to allow clients to choose the maximum number of issues pages that should be returned Adding new test cases for SearchGitHub.getAllProjectIssues(..) and SearchGitHub.getProjectIssues(..) --- .../cin/groundhog/search/SearchGitHub.java | 76 +++++++++++++++---- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 4fe782e..ff17bfa 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -42,7 +42,12 @@ */ public class SearchGitHub implements ForgeSearch { private static Logger logger = LoggerFactory.getLogger(SearchGitHub.class); - + + /* + * Default number of items per page as defined in http://developer.github.com/v3/#pagination + */ + private static final int DEFAULT_PAGINATION_LIMIT = 30; + public static int INFINITY = -1; private final Gson gson; @@ -251,7 +256,7 @@ public List getProjects(String term, String username, int page) throws for (JsonElement j: jsonArray) { Project p = gson.fromJson(j, Project.class); JsonObject jsonObj = j.getAsJsonObject(); - + p.setSCM(SCM.GIT); p.setScmURL(String.format("git@github.com:%s/%s.git", username, p.getName())); p.setSourceCodeURL(jsonObj.get("url").getAsString()); @@ -308,30 +313,71 @@ public List getProjectLanguages(Project project) { * @return a {@link List} of {@link Issues} objects */ public List getAllProjectIssues(Project project) { + //XXX: Instead fetching ALL issues at once we could return a lazy collection that would + // do it on the fly + int pageLimit = INFINITY; + return getProjectIssues(project, pageLimit); + } - logger.info("Searching project issues metadata"); - - String searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getSourceCodeURL().split("/")[3]) - .withSimpleParam("/", project.getName()) - .withParam("/issues") - .build(); - - String jsonString = requests.get(searchUrl); - JsonArray jsonArray = gson.fromJson(jsonString, JsonElement.class).getAsJsonArray(); + /** + * Returns all projects up to pageLimit pages. + * The default number of projects per page as defined by version V3 of the GitHub API is 30. + * @param project the project to fetch issues for + * @param pageLimit the maximum number of pages for which issues should be retrieved, use + * {@link #INFINITY} for all issues ever opened. + * @return + */ + public ListgetProjectIssues(Project project, int pageLimit) { + if(project == null){ + throw new IllegalArgumentException("project must not be null"); + } + if(pageLimit != INFINITY && pageLimit <= 0){ + throw new IllegalArgumentException("pageLimit must be > 0 or INFINITY"); + } + return getProjectIssues(project.getUser().getLogin().replace("\"", ""), + project.getName(), pageLimit); + } + private List getProjectIssues(String user, String project, int pageLimit) { List issues = new ArrayList(); + + JsonObject firstPageJson = getIssuesJson(user, project, 1); + JsonArray jsonArray = firstPageJson.get("items").getAsJsonArray(); for (JsonElement element : jsonArray) { Issue issue = gson.fromJson(element, Issue.class); - issue.setProject(project); issues.add(issue); } - + int totalItems = firstPageJson.get("total_count").getAsInt(); + int pages = ((totalItems - 1)/ DEFAULT_PAGINATION_LIMIT) + 1; + if(pageLimit != INFINITY){ + pages = Math.min(pages, pageLimit); + } + for(int i = 2; i <= pages; i++){ + JsonObject ithPageJson = getIssuesJson(user, project, i); + JsonArray ithJsonArray = ithPageJson.get("items").getAsJsonArray(); + for (JsonElement element : ithJsonArray) { + Issue issue = gson.fromJson(element, Issue.class); + issues.add(issue); + } + } return issues; } + private JsonObject getIssuesJson(String user, String project, int page) { + logger.info("Searching project issues metadata"); + String searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("search/issues") + .withParam("q", "user:" + user + "+" + "repo:" + project) + .withParam("page", "" + page) + .build(); + String jsonString = requests.get(searchUrl); + + JsonObject jsonObject = gson.fromJson(jsonString, JsonElement.class).getAsJsonObject(); + + return jsonObject; + } + /** * Fetches all the Milestones of the given {@link Project} from the GitHub API * @param project the @{link Project} of which the Milestones are about From f81ad819f5bec12d4ee3697e216f48755abdb5e5 Mon Sep 17 00:00:00 2001 From: Irineu Date: Wed, 8 Jan 2014 08:45:15 -0300 Subject: [PATCH 03/11] Rebasing on top of master --- .../br/ufpe/cin/groundhog/search/SearchGitHub.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index ff17bfa..b58c44f 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -334,18 +334,19 @@ public List getAllProjectIssues(Project project) { if(pageLimit != INFINITY && pageLimit <= 0){ throw new IllegalArgumentException("pageLimit must be > 0 or INFINITY"); } - return getProjectIssues(project.getUser().getLogin().replace("\"", ""), - project.getName(), pageLimit); + return getProjectIssuesInner(project, pageLimit); } - private List getProjectIssues(String user, String project, int pageLimit) { + private List getProjectIssuesInner(Project project, int pageLimit) { List issues = new ArrayList(); - JsonObject firstPageJson = getIssuesJson(user, project, 1); + JsonObject firstPageJson = getIssuesJson(project.getUser().getLogin(), + project.getName(), 1); JsonArray jsonArray = firstPageJson.get("items").getAsJsonArray(); for (JsonElement element : jsonArray) { Issue issue = gson.fromJson(element, Issue.class); + issue.setProject(project); issues.add(issue); } int totalItems = firstPageJson.get("total_count").getAsInt(); @@ -354,10 +355,12 @@ private List getProjectIssues(String user, String project, int pageLimit) pages = Math.min(pages, pageLimit); } for(int i = 2; i <= pages; i++){ - JsonObject ithPageJson = getIssuesJson(user, project, i); + JsonObject ithPageJson = getIssuesJson(project.getUser().getLogin(), + project.getName(), i); JsonArray ithJsonArray = ithPageJson.get("items").getAsJsonArray(); for (JsonElement element : ithJsonArray) { Issue issue = gson.fromJson(element, Issue.class); + issue.setProject(project); issues.add(issue); } } From e3e7abf3227c73382b5e7f67e0a051feed61bfb5 Mon Sep 17 00:00:00 2001 From: Irineu Date: Sun, 5 Jan 2014 12:01:28 -0300 Subject: [PATCH 04/11] Adding more metadata to commits Adding total additions and deletions count to Commits Adding new CommitFiles to Commits that enables client to retrieve the file names, per file addition/deletion/change counts, status and patch string --- .../main/br/ufpe/cin/groundhog/Commit.java | 44 ++++++++++++++++++- .../cin/groundhog/search/SearchGitHub.java | 26 ++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/Commit.java b/src/java/main/br/ufpe/cin/groundhog/Commit.java index bb1303d..3ab85a6 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Commit.java +++ b/src/java/main/br/ufpe/cin/groundhog/Commit.java @@ -1,6 +1,7 @@ package br.ufpe.cin.groundhog; import java.util.Date; +import java.util.List; import br.ufpe.cin.groundhog.util.Dates; @@ -35,6 +36,8 @@ public class Commit extends GitHubEntity { private int additionsCount; private int deletionsCount; + + private List files; public Commit(String sha, Project project) { this.sha = sha; @@ -133,7 +136,15 @@ public void setDeletionsCount(int deletionsCount) { public String getabbrevSHA() { return this.sha.substring(0, 7); } - + + public List getFiles() { + return this.files; + } + + public void setFiles(List files) { + this.files = files; + } + /** * Two {@link Commit} objects are considered equal if and only if both have the same SHA hash * @param commit @@ -159,4 +170,33 @@ public String getURL() { this.getProject().getName(), this.sha); } -} \ No newline at end of file + + public static final class CommitFile { + @SerializedName(value="filename") + private String fileName; + + @SerializedName(value="additions") + private int additionsCount; + + @SerializedName(value="deletions") + private int deletionsCount; + + @SerializedName(value="changes") + private int changesCount; + + @SerializedName(value="status") + private String status; + + @SerializedName(value="patch") + private String patch; + + @Override + public String toString() { + return "CommitFile [fileName=" + fileName + ", additionsCount=" + + additionsCount + ", deletionsCount=" + deletionsCount + + ", changesCount=" + changesCount + ", status=" + status + + ", patch=" + patch + "]"; + } + + } +} diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index b58c44f..773a260 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -10,6 +10,7 @@ import org.slf4j.LoggerFactory; import br.ufpe.cin.groundhog.Commit; +import br.ufpe.cin.groundhog.Commit.CommitFile; import br.ufpe.cin.groundhog.GroundhogException; import br.ufpe.cin.groundhog.Issue; import br.ufpe.cin.groundhog.Language; @@ -485,8 +486,31 @@ public List getAllProjectCommits(Project project) { String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); commit.setCommitDate(date); commits.add(commit); + + searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("repos") + .withSimpleParam("/", project.getUser().getLogin().replace("\"", "")) + .withSimpleParam("/", project.getName()) + .withParam("/commits") + .withSimpleParam("/", commit.getSha()) + .build(); + logger.info("searchUrl="+searchUrl); + jsonElement = gson.fromJson(requests.get(searchUrl), JsonElement.class); + JsonObject statsObject = jsonElement.getAsJsonObject().get("stats").getAsJsonObject(); + int additions = statsObject.get("additions").getAsInt(); + int deletions = statsObject.get("deletions").getAsInt(); + commit.setAdditionsCount(additions); + commit.setDeletionsCount(deletions); + JsonArray filesArray = jsonElement.getAsJsonObject().get("files").getAsJsonArray(); + List files = new ArrayList<>(); + for (JsonElement fileElement : filesArray) { + CommitFile gitFile = gson.fromJson(fileElement, CommitFile.class); + files.add(gitFile); + } + commit.setFiles(files); } + return commits; } @@ -728,4 +752,4 @@ private String getWithProtection(String url){ return data; } -} \ No newline at end of file +} From 679f26015bff3f93edef55c95b2a391a5189fb99 Mon Sep 17 00:00:00 2001 From: Irineu Date: Sat, 11 Jan 2014 10:13:09 -0300 Subject: [PATCH 05/11] Adding new API to Requests class to allow one to to get the Response object for a given request instead of simply the response body --- .../main/br/ufpe/cin/groundhog/http/Requests.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/java/main/br/ufpe/cin/groundhog/http/Requests.java b/src/java/main/br/ufpe/cin/groundhog/http/Requests.java index 105d6c9..e22a542 100644 --- a/src/java/main/br/ufpe/cin/groundhog/http/Requests.java +++ b/src/java/main/br/ufpe/cin/groundhog/http/Requests.java @@ -8,6 +8,7 @@ import com.ning.http.client.ListenableFuture; import com.ning.http.client.Request; import com.ning.http.client.RequestBuilder; +import com.ning.http.client.Response; /** * Utility class to perform asynchronous http requests @@ -35,6 +36,19 @@ public String get(String urlStr) { throw new HttpException(e); } } + + /** + * Gets the {@link Response} object for a given URL request + * @param urlStr the URL for the request to be performed + * @return the {@link Response} object for this request + */ + public Response getResponse(String urlStr) { + try { + return this.httpClient.prepareGet(urlStr).execute().get(); + } catch (Exception e) { + throw new HttpException(e); + } + } /** * Gets the response body of the given URL using the preview flag in the request header From 4cba833354dd7f0473ecb6e8cd3dd7e99d0c9b0e Mon Sep 17 00:00:00 2001 From: Irineu Date: Sat, 11 Jan 2014 12:58:03 -0300 Subject: [PATCH 06/11] Fixed issues search to only return issues for a given project and not all issues for a user Intermediate commit for the enhanced commits search API --- .../cin/groundhog/search/SearchGitHub.java | 292 +++++++++++------- 1 file changed, 186 insertions(+), 106 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 773a260..73d8b7c 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -2,9 +2,11 @@ import static br.ufpe.cin.groundhog.http.URLsDecoder.encodeURL; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,17 +25,22 @@ import br.ufpe.cin.groundhog.http.Requests; import br.ufpe.cin.groundhog.search.UrlBuilder.GithubAPI; +import static com.google.common.base.Preconditions.checkNotNull; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; import com.google.inject.Guice; import com.google.inject.Inject; +import com.ning.http.client.Response; import java.util.Collections; import java.util.HashMap; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Performs the project search on GitHub, via its official JSON API @@ -42,13 +49,20 @@ * @since 0.0.1 */ public class SearchGitHub implements ForgeSearch { + private static Logger logger = LoggerFactory.getLogger(SearchGitHub.class); /* * Default number of items per page as defined in http://developer.github.com/v3/#pagination */ + @SuppressWarnings("unused") private static final int DEFAULT_PAGINATION_LIMIT = 30; + /* + * Pattern for a Github Link header + */ + private static final Pattern LINK_PATTERN = Pattern.compile("\\<(?\\S+)\\>\\; rel=\\\"(?\\S+)\\\""); + public static int INFINITY = -1; private final Gson gson; @@ -308,7 +322,7 @@ public List getProjectLanguages(Project project) { } /** - * Fetches all the Issues of the given {@link Project} from the GitHub API + * Fetches all the Issues (open and closed) of the given {@link Project} from the GitHub API * * @param project the @{link Project} of which the Issues are about * @return a {@link List} of {@link Issues} objects @@ -316,70 +330,85 @@ public List getProjectLanguages(Project project) { public List getAllProjectIssues(Project project) { //XXX: Instead fetching ALL issues at once we could return a lazy collection that would // do it on the fly - int pageLimit = INFINITY; - return getProjectIssues(project, pageLimit); + try { + return getProjectIssuesInner(project); + } catch (JsonSyntaxException | IOException e) { + throw new SearchException(e); + } } - /** - * Returns all projects up to pageLimit pages. - * The default number of projects per page as defined by version V3 of the GitHub API is 30. - * @param project the project to fetch issues for - * @param pageLimit the maximum number of pages for which issues should be retrieved, use - * {@link #INFINITY} for all issues ever opened. - * @return - */ - public ListgetProjectIssues(Project project, int pageLimit) { - if(project == null){ - throw new IllegalArgumentException("project must not be null"); - } - if(pageLimit != INFINITY && pageLimit <= 0){ - throw new IllegalArgumentException("pageLimit must be > 0 or INFINITY"); - } - return getProjectIssuesInner(project, pageLimit); + private List getProjectIssuesInner(Project project) throws JsonSyntaxException, IOException { + logger.info("Searching project issues metadata"); + List issues = getAllProjectIssuesForState(project, "open"); + issues.addAll(getAllProjectIssuesForState(project, "closed")); + return issues; } - private List getProjectIssuesInner(Project project, int pageLimit) { + private List getAllProjectIssuesForState(Project project, + String state) throws IOException { List issues = new ArrayList(); - - - JsonObject firstPageJson = getIssuesJson(project.getUser().getLogin(), - project.getName(), 1); - JsonArray jsonArray = firstPageJson.get("items").getAsJsonArray(); + String searchUrl = buildListIssuesUrl(project.getUser().getLogin(), project.getName(), 1, state); + Response response = getResponseWithProtection(searchUrl); + String jsonResponse = response.getResponseBody(); + extractIssues(project, jsonResponse, issues); + while((searchUrl = getNextUrl(response)) != null){ + response = getResponseWithProtection(searchUrl); + jsonResponse = response.getResponseBody(); + extractIssues(project, jsonResponse, issues); + } + return issues; + } + + private void extractIssues(Project project, String jsonResponse, + List issues) { + JsonArray jsonArray = gson.fromJson(jsonResponse, JsonElement.class).getAsJsonArray(); for (JsonElement element : jsonArray) { Issue issue = gson.fromJson(element, Issue.class); issue.setProject(project); issues.add(issue); } - int totalItems = firstPageJson.get("total_count").getAsInt(); - int pages = ((totalItems - 1)/ DEFAULT_PAGINATION_LIMIT) + 1; - if(pageLimit != INFINITY){ - pages = Math.min(pages, pageLimit); - } - for(int i = 2; i <= pages; i++){ - JsonObject ithPageJson = getIssuesJson(project.getUser().getLogin(), - project.getName(), i); - JsonArray ithJsonArray = ithPageJson.get("items").getAsJsonArray(); - for (JsonElement element : ithJsonArray) { - Issue issue = gson.fromJson(element, Issue.class); - issue.setProject(project); - issues.add(issue); - } - } - return issues; } - private JsonObject getIssuesJson(String user, String project, int page) { - logger.info("Searching project issues metadata"); + private String buildListIssuesUrl(String user, String project, int page, String state){ String searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("search/issues") - .withParam("q", "user:" + user + "+" + "repo:" + project) + .withParam("repos") + .withSimpleParam("/", user) + .withSimpleParam("/", project) + .withSimpleParam("/", "issues") + .withParam("state", state) .withParam("page", "" + page) .build(); - String jsonString = requests.get(searchUrl); - - JsonObject jsonObject = gson.fromJson(jsonString, JsonElement.class).getAsJsonObject(); + return searchUrl; + } - return jsonObject; + private String getNextUrl(Response response) { + String nextLink = null; + String linkHeader = response.getHeader("Link"); + if(linkHeader != null){ + Map relToLinks = getLinks(linkHeader); + if(relToLinks.containsKey("next")){ + nextLink = relToLinks.get("next"); + } + } + return nextLink; + } + + /* + * Extract the links out of a Link header + */ + private Map getLinks(String linkHeader) { + Map relToLinks = new HashMap<>(); + Matcher linkMatcher = LINK_PATTERN.matcher(linkHeader); + while(linkMatcher.find()){ + String rel = linkMatcher.group("rel"); + String link = linkMatcher.group("link"); + if(!relToLinks.containsKey(rel)) relToLinks.put(rel, link); + else { + logger.warn("Duplicate rel, previous link " + relToLinks.get(rel) + + " , new link " + link); + } + } + return relToLinks; } /** @@ -459,18 +488,55 @@ public int getNumberProjectTags(Project project) { * @return a {@link List} of {@link Commit} objects */ public List getAllProjectCommits(Project project) { - logger.info("Searching project commits metadata"); - - String searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getSourceCodeURL().split("/")[3]) - .withSimpleParam("/", project.getName()) - .withParam("/commits") - .build(); + checkNotNull(project, "project must not be null"); + logger.info("Searching all project commits metadata"); + return getProjectCommits(project, null, null); + } + + /** + * Fetches all the Commits of the given {@link Project} from the GitHub API for the specified + * period. If either since or until are null the period is open-ended, if both are + * null then this method behaves exactly like {@link #getAllProjectCommits(Project)} + * @param project project to fetch commits metadata for + * @param since initial date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @param until end date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsByPeriod(Project project, String since, String until){ + checkNotNull(project, "project must not be null"); + logger.info("Searching project commits metadata by period"); + return getProjectCommits(project, since, until); + } + + /** + * Utility method for getAllProjectCommitsByPeriod(project, since, null) + * @param project project to fetch commits metadata for + * @param since initial date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsSince(Project project, String since){ + return getProjectCommitsByPeriod(project, since, null); + } + + /** + * Utility method for getAllProjectCommitsByPeriod(project, null, until) + * @param project project to fetch commits metadata for + * @param until end date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsUntil(Project project, String until){ + return getProjectCommitsByPeriod(project, null, until); + } + + private List getProjectCommits(Project project, String since, String until) { + String searchUrl = buildListCommitsUrl(project, since, until); - System.out.println(searchUrl); + String response = getWithProtection(searchUrl); - JsonElement jsonElement = gson.fromJson(requests.get(searchUrl), JsonElement.class); + JsonElement jsonElement = gson.fromJson(response, JsonElement.class); JsonArray jsonArray = jsonElement.getAsJsonArray(); List commits = new ArrayList<>(); @@ -487,67 +553,61 @@ public List getAllProjectCommits(Project project) { commit.setCommitDate(date); commits.add(commit); - searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getUser().getLogin().replace("\"", "")) - .withSimpleParam("/", project.getName()) - .withParam("/commits") - .withSimpleParam("/", commit.getSha()) - .build(); - logger.info("searchUrl="+searchUrl); - jsonElement = gson.fromJson(requests.get(searchUrl), JsonElement.class); - JsonObject statsObject = jsonElement.getAsJsonObject().get("stats").getAsJsonObject(); - int additions = statsObject.get("additions").getAsInt(); - int deletions = statsObject.get("deletions").getAsInt(); - commit.setAdditionsCount(additions); - commit.setDeletionsCount(deletions); - JsonArray filesArray = jsonElement.getAsJsonObject().get("files").getAsJsonArray(); - List files = new ArrayList<>(); - for (JsonElement fileElement : filesArray) { - CommitFile gitFile = gson.fromJson(fileElement, CommitFile.class); - files.add(gitFile); - } - commit.setFiles(files); + populateRemainingMetadata(commit); } - return commits; } - /** - * Fetches all the Commits of the given {@link Project} from the GitHub API - * @param project the @{link Project} to which the commits belong - * @return a {@link List} of {@link Commit} objects - */ - public List getAllProjectCommitsByDate(Project project, String start, String end) { - - logger.info("Searching all project commits metadata by date"); - - String searchUrl = builder.uses(GithubAPI.ROOT) + private String buildListCommitsUrl(Project project, String since, + String until) { + UrlBuilder listCommitsBuilder = builder.uses(GithubAPI.ROOT) .withParam("repos") .withSimpleParam("/", project.getUser().getLogin()) .withSimpleParam("/", project.getName()) - .withSimpleParam("/", "commits") - .withParam("since", start) - .withParam("until", end) - .build(); + .withSimpleParam("/", "commits"); - String response = getWithProtection(searchUrl); + if(since != null){ + listCommitsBuilder = listCommitsBuilder.withParam("since", since); + } - JsonElement jsonElement = gson.fromJson(response, JsonElement.class); - JsonArray jsonArray = jsonElement.getAsJsonArray(); + if(until != null) { + listCommitsBuilder = listCommitsBuilder.withParam("until", until); + } - List commits = new ArrayList<>(); - for (JsonElement element : jsonArray) { - Commit commit = gson.fromJson(element, Commit.class); - commit.setProject(project); + String searchUrl = listCommitsBuilder.build(); + return searchUrl; + } - String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); - commit.setCommitDate(date); - commits.add(commit); + private void populateRemainingMetadata(Commit commit) { + JsonElement jsonElement; + String searchUrl = buildCommitSearchUrl(commit); + jsonElement = gson.fromJson(getWithProtection(searchUrl), JsonElement.class); + JsonObject statsObject = jsonElement.getAsJsonObject().get("stats").getAsJsonObject(); + int additions = statsObject.get("additions").getAsInt(); + int deletions = statsObject.get("deletions").getAsInt(); + commit.setAdditionsCount(additions); + commit.setDeletionsCount(deletions); + JsonArray filesArray = jsonElement.getAsJsonObject().get("files").getAsJsonArray(); + List files = new ArrayList<>(); + for (JsonElement fileElement : filesArray) { + CommitFile gitFile = gson.fromJson(fileElement, CommitFile.class); + files.add(gitFile); } + commit.setFiles(files); + } - return commits; + private String buildCommitSearchUrl(Commit commit) { + String searchUrl; + Project project = commit.getProject(); + searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("repos") + .withSimpleParam("/", project.getUser().getLogin()) + .withSimpleParam("/", project.getName()) + .withParam("/commits") + .withSimpleParam("/", commit.getSha()) + .build(); + return searchUrl; } /** @@ -752,4 +812,24 @@ private String getWithProtection(String url){ return data; } + + private Response getResponseWithProtection(String url){ + Response response = requests.getResponse(url); + String data; + try { + data = response.getResponseBody(); + if(data != null && data.contains("API rate limit exceeded for")) { + try { + Thread.sleep(1000 * 60 * 60); + data = requests.get(url); + + } catch (InterruptedException ex) { + ex.printStackTrace(); + } + } + } catch (IOException e) { + throw new SearchException(e); + } + return response; + } } From 52e189be5900b0ee4eae5565f3436a2619c0355e Mon Sep 17 00:00:00 2001 From: Irineu Date: Sun, 12 Jan 2014 13:58:22 -0300 Subject: [PATCH 07/11] Enabling retrieval all commits for a given commits search Removing test case for removed API --- .../cin/groundhog/search/SearchGitHub.java | 54 +++++++++++++------ .../groundhog/search/SearchGitHubTest.java | 19 ------- 2 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 73d8b7c..d3cb4d3 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -1,12 +1,18 @@ package br.ufpe.cin.groundhog.search; import static br.ufpe.cin.groundhog.http.URLsDecoder.encodeURL; +import static com.google.common.base.Preconditions.checkNotNull; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +31,6 @@ import br.ufpe.cin.groundhog.http.Requests; import br.ufpe.cin.groundhog.search.UrlBuilder.GithubAPI; -import static com.google.common.base.Preconditions.checkNotNull; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -36,12 +41,6 @@ import com.google.inject.Inject; import com.ning.http.client.Response; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map.Entry; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /** * Performs the project search on GitHub, via its official JSON API * @@ -490,7 +489,11 @@ public int getNumberProjectTags(Project project) { public List getAllProjectCommits(Project project) { checkNotNull(project, "project must not be null"); logger.info("Searching all project commits metadata"); - return getProjectCommits(project, null, null); + try { + return getProjectCommits(project, null, null); + } catch (IOException e) { + throw new SearchException(e); + } } /** @@ -506,7 +509,11 @@ public List getAllProjectCommits(Project project) { public List getProjectCommitsByPeriod(Project project, String since, String until){ checkNotNull(project, "project must not be null"); logger.info("Searching project commits metadata by period"); - return getProjectCommits(project, since, until); + try { + return getProjectCommits(project, since, until); + } catch (IOException e) { + throw new SearchException(e); + } } /** @@ -531,15 +538,31 @@ public List getProjectCommitsUntil(Project project, String until){ return getProjectCommitsByPeriod(project, null, until); } - private List getProjectCommits(Project project, String since, String until) { + private List getProjectCommits(Project project, String since, String until) throws IOException { + List commits = new ArrayList<>(); + int page = 1; String searchUrl = buildListCommitsUrl(project, since, until); + logger.info("getting commits for page " + page); + logger.info("searchUrl=" + searchUrl); + Response response = getResponseWithProtection(searchUrl); + extractCommits(project, response, commits); - String response = getWithProtection(searchUrl); + while((searchUrl = getNextUrl(response)) != null){ + logger.info("getting commits for page " + ++page); + response = getResponseWithProtection(searchUrl); + extractCommits(project, response, commits); + logger.info("done extracting page " + page); + } + + return commits; + } - JsonElement jsonElement = gson.fromJson(response, JsonElement.class); + private void extractCommits(Project project, Response response, + List commits) throws IOException { + String jsonResponse = response.getResponseBody(); + JsonElement jsonElement = gson.fromJson(jsonResponse, JsonElement.class); JsonArray jsonArray = jsonElement.getAsJsonArray(); - List commits = new ArrayList<>(); for (JsonElement element : jsonArray) { Commit commit = gson.fromJson(element, Commit.class); commit.setProject(project); @@ -551,12 +574,11 @@ private List getProjectCommits(Project project, String since, String unt String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); commit.setCommitDate(date); - commits.add(commit); populateRemainingMetadata(commit); - } - return commits; + commits.add(commit); + } } private String buildListCommitsUrl(Project project, String since, diff --git a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java index 1d16a0a..e1e039d 100644 --- a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java +++ b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java @@ -120,25 +120,6 @@ public void testGetAllProjectReleases() { } } - @Test - public void testGetProjectIssues(){ - try { - User u = new User("spgroup"); - Project project = new Project(u, "groundhog"); - - List projectIssues = searchGitHub.getProjectIssues(project, 2); - - Assert.assertNotNull(projectIssues); - // The default number of items per page is 30 so for 2 pages we should have 60 items - // As of 01-Jan-2014 the number of issues for project spgroup/groundhog is 315 so it - // should be guaranteed to have 2 pages at least. - Assert.assertEquals(projectIssues.size(), 60); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); - } - } - // Note that in the future this test may have to be reworked as the number of issues in the // target project grows @Test From e00b03b90336bfe8017f7d2d1ef55839120dd47c Mon Sep 17 00:00:00 2001 From: Irineu Date: Sun, 12 Jan 2014 22:39:58 -0300 Subject: [PATCH 08/11] Fixing bug that caused the getResponseWithProtection and getWithProtection methods to erroneously sleep for the predefined time if the body of a response contained the "API rate limit exceeded for" string even if the response was not a 403 - Forbidden due to an actual rate limit overage. --- .../cin/groundhog/search/SearchGitHub.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index d3cb4d3..924fa40 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -820,31 +820,27 @@ public List getAllProjectReleases(Project project) { } private String getWithProtection(String url){ - String data = requests.get(url); - - if (data.contains("API rate limit exceeded for")) { - try { - Thread.sleep(1000 * 60 * 60); - data = requests.get(url); - - } catch (InterruptedException ex) { - ex.printStackTrace(); - } + try { + return getResponseWithProtection(url).getResponseBody(); + } catch (IOException e) { + // This should never happen, but let's be safe + throw new SearchException(e); } - - return data; } private Response getResponseWithProtection(String url){ Response response = requests.getResponse(url); String data; try { + int statusCode = response.getStatusCode(); + data = response.getResponseBody(); - if(data != null && data.contains("API rate limit exceeded for")) { + // 403 == forbidden + if(statusCode == 403 && data != null && data.contains("API rate limit exceeded")) { try { + logger.info("API rate limit exceeded, waiting for " + (60 * 60) + " seconds"); Thread.sleep(1000 * 60 * 60); data = requests.get(url); - } catch (InterruptedException ex) { ex.printStackTrace(); } From ced197de8c91dbbfc01e0ec40cdb4b0dc3f49ff4 Mon Sep 17 00:00:00 2001 From: Irineu Date: Mon, 27 Jan 2014 20:38:18 -0300 Subject: [PATCH 09/11] Finishing implementation of issue labels fetching --- .../main/br/ufpe/cin/groundhog/Issue.java | 598 +++++++++--------- 1 file changed, 299 insertions(+), 299 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/Issue.java b/src/java/main/br/ufpe/cin/groundhog/Issue.java index bce06e2..b41813d 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Issue.java +++ b/src/java/main/br/ufpe/cin/groundhog/Issue.java @@ -1,300 +1,300 @@ -package br.ufpe.cin.groundhog; - -import java.util.Date; -import java.util.List; - -import org.mongodb.morphia.annotations.Entity; -import org.mongodb.morphia.annotations.Indexed; -import org.mongodb.morphia.annotations.Reference; - -import com.google.gson.annotations.SerializedName; - -/** - * Represents an Issue object in Groundhog - * @author Rodrigo Alves - */ -@Entity("issues") -public class Issue extends GitHubEntity { - @Indexed(unique=true, dropDups=true) - @SerializedName("id") - private int id; - - @SerializedName("number") - private int number; - - @SerializedName("comments") - private int commentsCount; - - @Reference private Project project; - - @SerializedName("pull_request") - private transient PullRequest pullRequest; - - private List labels; - - @Reference private Milestone milestone; - - @SerializedName("title") - private String title; - - @SerializedName("body") - private String body; - - @SerializedName("state") - private String state; - - @SerializedName("assignee") - private User assignee; - - @SerializedName("closed_by") - private User closedBy; - - @SerializedName("created_at") - private Date createdAt; - - @SerializedName("updated_at") - private Date updatedAt; - - @SerializedName("closed_at") - private Date closedAt; - - public Issue(Project project, int number, String state) { - this.number = number; - this.project = project; - this.state = state; - } - - public Issue(Project project, int number, String state, String title) { - this(project, number, state); - this.title = title; - } - - /** - * Returns the ID of the Issue on GitHub. This ID is unique for every Issue on GitHub, which means that - * no two (or more) Issues on GitHub may have the same ID - * @return - */ - public int getId() { - return this.id; - } - - public void setId(int id) { - this.id = id; - } - - /** - * Returns the number of the Issue on its Project. This number is unique within the project to which - * the Issues belong. Thus, the same project may not have two Issues with the same number but two (or more) - * different projects on GitHub may have Issues with the same number. - * @return - */ - public int getNumber() { - return this.number; - } - - public void setNumber(int number) { - this.number = number; - } - - /** - * Informs the number of comments on the Issue - * @return - */ - public int getCommentsCount() { - return this.commentsCount; - } - - public void setCommentsCount(int commentsCount) { - this.commentsCount = commentsCount; - } - - /** - * Informs the Project object to which the Issue belongs - * No Issue exists without a project - * @return - */ - public Project getProject() { - return this.project; - } - - public void setProject(Project project) { - this.project = project; - } - - /** - * Informs the PullRequest related to the Issue. Not all Issues are Pull Request Issues. - * If this method return nulls then it means the Issue is not a PullRequest Issue - * @return - */ - public PullRequest getPullRequest() { - return this.pullRequest; - } - - public void setPullRequest(PullRequest pullRequest) { - this.pullRequest = pullRequest; - } - - public List getLabels() { - return this.labels; - } - - public void setLabels(List labels) { - this.labels = labels; - } - - public Milestone getMilestone() { - return this.milestone; - } - - public void setMilestone(Milestone milestone) { - this.milestone = milestone; - } - - /** - * Returns the title of the Issue - * @return - */ - public String getTitle() { - return this.title; - } - - public void setTitle(String title) { - this.title = title; - } - - /** - * Returns the Markdown-syntax-based description of the Issue - * @return - */ - public String getBody() { - return this.body; - } - - public void setBody(String body) { - this.body = body; - } - - /** - * Returns the current state of the Issue. Possible values are "open" and "closed" - * @return - */ - public String getState() { - return this.state; - } - - public void setState(String state) { - this.state = state; - } - - /** - * Returns the User assigned to that Issue. - * An Issue on GitHub may or may not have an assignee - * @return - */ - public User getAssignee() { - return this.assignee; - } - - public void setAssignee(User assignee) { - this.assignee = assignee; - } - - /** - * Informs the User who closed the Issue - * Every Issue gets closed by someone, so if this value is null - * then the Issue is currently open - * @return - */ - public User getClosedBy() { - return this.closedBy; - } - - public void setClosedBy(User closedBy) { - this.closedBy = closedBy; - } - - /** - * Informs the creation date of the Issue on GitHub - * @return - */ - public Date getCreatedAt() { - return this.createdAt; - } - - public void setCreatedAt(Date createdAt) { - this.createdAt = createdAt; - } - - /** - * Informs the date when the last modification was made upon the issue - * @return - */ - public Date getUpdatedAt() { - return this.updatedAt; - } - - public void setUpdatedAt(Date updatedAt) { - this.updatedAt = updatedAt; - } - - /** - * Informs the date when the Issue was closed - * If the value is null it means the issue is open - * @return - */ - public Date getClosedAt() { - return this.closedAt; - } - - public void setClosedAt(Date closedAt) { - this.closedAt = closedAt; - } - - /** - * Returns true if the Issue is open. Returns false otherwise - * @return - */ - public boolean isOpen() { - return this.getState() == "closed" ? true : false; - } - - /** - * Returns true if the Issue is a Pull Request Issue. Returns false otherwise - * @return - */ - public boolean isPullRequest() { - return this.getPullRequest() != null ? true : false; - } - - /** - * Two {@link Issue} objects are considered equal when they have the same GitHub API ID, - * the same number and the same {@link Project}. - * @param issue - * @return - */ - public boolean equals(Issue issue) { - return this.id == issue.id && this.number == issue.number && this.project.equals(issue.getProject()); - } - - public String getURL() { - return String.format("https://api.github.com/repos/%s/%s/issues/%d", - this.getProject().getOwner().getLogin(), this.getProject().getName(), this.getNumber()); - } - - @Override - public String toString() { - String stringReturn = "Issue Number = " + this.number; - - if (this.title != null) { - stringReturn += ", title: " + this.title; - } - - String url = this.getURL(); // This class doesn't contains a variable referring a URL, so we create one local - - if (url != null) { - stringReturn += ", URL = " + url; - } - - return stringReturn; - } +package br.ufpe.cin.groundhog; + +import java.util.Date; + +import org.mongodb.morphia.annotations.Entity; +import org.mongodb.morphia.annotations.Indexed; +import org.mongodb.morphia.annotations.Reference; + +import com.google.gson.annotations.SerializedName; + +/** + * Represents an Issue object in Groundhog + * @author Rodrigo Alves + */ +@Entity("issues") +public class Issue extends GitHubEntity { + @Indexed(unique=true, dropDups=true) + @SerializedName("id") + private int id; + + @SerializedName("number") + private int number; + + @SerializedName("comments") + private int commentsCount; + + @Reference private Project project; + + @SerializedName("pull_request") + private transient PullRequest pullRequest; + + @SerializedName("labels") + private IssueLabel[] labels; + + @Reference private Milestone milestone; + + @SerializedName("title") + private String title; + + @SerializedName("body") + private String body; + + @SerializedName("state") + private String state; + + @SerializedName("assignee") + private User assignee; + + @SerializedName("closed_by") + private User closedBy; + + @SerializedName("created_at") + private Date createdAt; + + @SerializedName("updated_at") + private Date updatedAt; + + @SerializedName("closed_at") + private Date closedAt; + + public Issue(Project project, int number, String state) { + this.number = number; + this.project = project; + this.state = state; + } + + public Issue(Project project, int number, String state, String title) { + this(project, number, state); + this.title = title; + } + + /** + * Returns the ID of the Issue on GitHub. This ID is unique for every Issue on GitHub, which means that + * no two (or more) Issues on GitHub may have the same ID + * @return + */ + public int getId() { + return this.id; + } + + public void setId(int id) { + this.id = id; + } + + /** + * Returns the number of the Issue on its Project. This number is unique within the project to which + * the Issues belong. Thus, the same project may not have two Issues with the same number but two (or more) + * different projects on GitHub may have Issues with the same number. + * @return + */ + public int getNumber() { + return this.number; + } + + public void setNumber(int number) { + this.number = number; + } + + /** + * Informs the number of comments on the Issue + * @return + */ + public int getCommentsCount() { + return this.commentsCount; + } + + public void setCommentsCount(int commentsCount) { + this.commentsCount = commentsCount; + } + + /** + * Informs the Project object to which the Issue belongs + * No Issue exists without a project + * @return + */ + public Project getProject() { + return this.project; + } + + public void setProject(Project project) { + this.project = project; + } + + /** + * Informs the PullRequest related to the Issue. Not all Issues are Pull Request Issues. + * If this method return nulls then it means the Issue is not a PullRequest Issue + * @return + */ + public PullRequest getPullRequest() { + return this.pullRequest; + } + + public void setPullRequest(PullRequest pullRequest) { + this.pullRequest = pullRequest; + } + + public IssueLabel[] getLabels() { + return this.labels; + } + + public void setLabels(IssueLabel[] labels) { + this.labels = labels; + } + + public Milestone getMilestone() { + return this.milestone; + } + + public void setMilestone(Milestone milestone) { + this.milestone = milestone; + } + + /** + * Returns the title of the Issue + * @return + */ + public String getTitle() { + return this.title; + } + + public void setTitle(String title) { + this.title = title; + } + + /** + * Returns the Markdown-syntax-based description of the Issue + * @return + */ + public String getBody() { + return this.body; + } + + public void setBody(String body) { + this.body = body; + } + + /** + * Returns the current state of the Issue. Possible values are "open" and "closed" + * @return + */ + public String getState() { + return this.state; + } + + public void setState(String state) { + this.state = state; + } + + /** + * Returns the User assigned to that Issue. + * An Issue on GitHub may or may not have an assignee + * @return + */ + public User getAssignee() { + return this.assignee; + } + + public void setAssignee(User assignee) { + this.assignee = assignee; + } + + /** + * Informs the User who closed the Issue + * Every Issue gets closed by someone, so if this value is null + * then the Issue is currently open + * @return + */ + public User getClosedBy() { + return this.closedBy; + } + + public void setClosedBy(User closedBy) { + this.closedBy = closedBy; + } + + /** + * Informs the creation date of the Issue on GitHub + * @return + */ + public Date getCreatedAt() { + return this.createdAt; + } + + public void setCreatedAt(Date createdAt) { + this.createdAt = createdAt; + } + + /** + * Informs the date when the last modification was made upon the issue + * @return + */ + public Date getUpdatedAt() { + return this.updatedAt; + } + + public void setUpdatedAt(Date updatedAt) { + this.updatedAt = updatedAt; + } + + /** + * Informs the date when the Issue was closed + * If the value is null it means the issue is open + * @return + */ + public Date getClosedAt() { + return this.closedAt; + } + + public void setClosedAt(Date closedAt) { + this.closedAt = closedAt; + } + + /** + * Returns true if the Issue is open. Returns false otherwise + * @return + */ + public boolean isOpen() { + return this.getState() == "closed" ? true : false; + } + + /** + * Returns true if the Issue is a Pull Request Issue. Returns false otherwise + * @return + */ + public boolean isPullRequest() { + return this.getPullRequest() != null ? true : false; + } + + /** + * Two {@link Issue} objects are considered equal when they have the same GitHub API ID, + * the same number and the same {@link Project}. + * @param issue + * @return + */ + public boolean equals(Issue issue) { + return this.id == issue.id && this.number == issue.number && this.project.equals(issue.getProject()); + } + + public String getURL() { + return String.format("https://api.github.com/repos/%s/%s/issues/%d", + this.getProject().getOwner().getLogin(), this.getProject().getName(), this.getNumber()); + } + + @Override + public String toString() { + String stringReturn = "Issue Number = " + this.number; + + if (this.title != null) { + stringReturn += ", title: " + this.title; + } + + String url = this.getURL(); // This class doesn't contains a variable referring a URL, so we create one local + + if (url != null) { + stringReturn += ", URL = " + url; + } + + return stringReturn; + } } \ No newline at end of file From 19360ed80c9c9d2938216349818a14a6215f6c78 Mon Sep 17 00:00:00 2001 From: Irineu Date: Wed, 29 Jan 2014 22:17:12 -0300 Subject: [PATCH 10/11] Greatly improving project commit search and adding more commit metadata --- .../main/br/ufpe/cin/groundhog/Commit.java | 142 ++++++++++++---- .../cin/groundhog/search/SearchGitHub.java | 159 +++++++++++------- 2 files changed, 203 insertions(+), 98 deletions(-) diff --git a/src/java/main/br/ufpe/cin/groundhog/Commit.java b/src/java/main/br/ufpe/cin/groundhog/Commit.java index 3ab85a6..a290299 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Commit.java +++ b/src/java/main/br/ufpe/cin/groundhog/Commit.java @@ -33,11 +33,11 @@ public class Commit extends GitHubEntity { private Date commitDate; - private int additionsCount; - - private int deletionsCount; + private CommitStats stats; private List files; + + private List parents; public Commit(String sha, Project project) { this.sha = sha; @@ -105,30 +105,6 @@ public void setCommitDate(String date) { this.commitDate = createAtDate; } - /** - * Informs the sum of added lines among the files committed - * @param deletionsCount - */ - public int getAdditionsCount() { - return this.additionsCount; - } - - public void setAdditionsCount(int additionsCount) { - this.additionsCount = additionsCount; - } - - /** - * Informs the sum of deleted lines among the files committed - * @param deletionsCount - */ - public int getDeletionsCount() { - return this.deletionsCount; - } - - public void setDeletionsCount(int deletionsCount) { - this.deletionsCount = deletionsCount; - } - /** * Gives the abbreviated SHA of the {@link Commit} object * @return a {@link String} object @@ -141,8 +117,20 @@ public List getFiles() { return this.files; } - public void setFiles(List files) { - this.files = files; + public CommitStats getStats() { + return this.stats; + } + + public List getParents() { + return this.parents; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((sha == null) ? 0 : sha.hashCode()); + return result; } /** @@ -150,10 +138,23 @@ public void setFiles(List files) { * @param commit * @return */ - public boolean equals(Commit commit) { - return this.sha == commit.sha; + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Commit other = (Commit) obj; + if (sha == null) { + if (other.sha != null) + return false; + } else if (!sha.equals(other.sha)) + return false; + return true; } - + @Override public String toString() { return "Commit [" + (sha != null ? "sha=" + sha + ", " : "") @@ -171,6 +172,38 @@ public String getURL() { this.sha); } + /** + * The commit changes stats, i.e., number of additions, deletions and total + * @author Irineu + * + */ + public static final class CommitStats { + private int additions; + + private int deletions; + + private int total; + + public int getAdditions() { + return additions; + } + + public int getDeletions() { + return deletions; + } + + public int getTotal() { + return total; + } + + @Override + public String toString() { + return "CommitStats [additions=" + additions + ", deletions=" + + deletions + ", total=" + total + "]"; + } + + } + public static final class CommitFile { @SerializedName(value="filename") private String fileName; @@ -189,7 +222,31 @@ public static final class CommitFile { @SerializedName(value="patch") private String patch; - + + public String getFileName() { + return fileName; + } + + public int getAdditionsCount() { + return additionsCount; + } + + public int getDeletionsCount() { + return deletionsCount; + } + + public int getChangesCount() { + return changesCount; + } + + public String getStatus() { + return status; + } + + public String getPatch() { + return patch; + } + @Override public String toString() { return "CommitFile [fileName=" + fileName + ", additionsCount=" @@ -197,6 +254,25 @@ public String toString() { + ", changesCount=" + changesCount + ", status=" + status + ", patch=" + patch + "]"; } + } + + public static class CommitParent { + + private String url; + + private String sha; + public String getUrl() { + return url; + } + + public String getSha() { + return sha; + } + + @Override + public String toString() { + return "CommitParent [url=" + url + ", sha=" + sha + "]"; + } } } diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 924fa40..ec5306b 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -18,7 +18,6 @@ import org.slf4j.LoggerFactory; import br.ufpe.cin.groundhog.Commit; -import br.ufpe.cin.groundhog.Commit.CommitFile; import br.ufpe.cin.groundhog.GroundhogException; import br.ufpe.cin.groundhog.Issue; import br.ufpe.cin.groundhog.Language; @@ -482,18 +481,67 @@ public int getNumberProjectTags(Project project) { } /** - * Fetches all the Commits of the given {@link Project} from the GitHub API - * @param project the @{link Project} to which the commits belong - * @return a {@link List} of {@link Commit} objects + * Returns the commit with the given SHA sha for project + * @param project + * @param sha + * @return */ - public List getAllProjectCommits(Project project) { - checkNotNull(project, "project must not be null"); - logger.info("Searching all project commits metadata"); - try { - return getProjectCommits(project, null, null); - } catch (IOException e) { - throw new SearchException(e); + public Commit getProjectCommit(Project project, String sha) { + checkNotNull(project); + checkNotNull(sha); + Commit commit = null; + String searchUrl = buildCommitSearchUrl(project, sha); + String responseBody = getWithProtection(searchUrl); + commit = gson.fromJson(responseBody, Commit.class); + return commit; + } + + private String buildCommitSearchUrl(Project project, String sha) { + String searchUrl; + searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("repos") + .withSimpleParam("/", project.getUser().getLogin()) + .withSimpleParam("/", project.getName()) + .withParam("/commits") + .withSimpleParam("/", sha) + .build(); + return searchUrl; + } + + /** + * Returns the commit pointed by refSpec for Project project. + * @param project + * @return + */ + public Commit getProjectCommitByRef(Project project, String refSpec){ + checkNotNull(project); + checkNotNull(refSpec); + Commit head = null; + String refSHA = getRefSHA(project, refSpec); + if(refSHA != null) { + head = getProjectCommit(project, refSHA); } + return head; + } + + private String getRefSHA(Project project, String refName) { + String sha = null; + String searchUrl = buildRefSearchUrl(project, refName); + String responseBody = getWithProtection(searchUrl); + JsonElement ref = gson.fromJson(responseBody, JsonElement.class); + JsonElement object = ref.getAsJsonObject().get("object"); + sha = object.getAsJsonObject().get("sha").getAsString(); + return sha; + } + + private String buildRefSearchUrl(Project project, String refName) { + return builder.uses(GithubAPI.ROOT). + withParam("repos"). + withSimpleParam("/", project.getUser().getLogin()). + withSimpleParam("/", project.getName()). + withSimpleParam("/", "git"). + withSimpleParam("/", "refs"). + withSimpleParam("/", refName).build(); } /** @@ -538,12 +586,26 @@ public List getProjectCommitsUntil(Project project, String until){ return getProjectCommitsByPeriod(project, null, until); } + /** + * Fetches all the Commits of the given {@link Project} from the GitHub API + * @param project the @{link Project} to which the commits belong + * @return a {@link List} of {@link Commit} objects + */ + public List getAllProjectCommits(Project project) { + checkNotNull(project, "project must not be null"); + logger.info("Searching all project commits metadata"); + try { + return getProjectCommits(project, null, null); + } catch (IOException e) { + throw new SearchException(e); + } + } + private List getProjectCommits(Project project, String since, String until) throws IOException { List commits = new ArrayList<>(); int page = 1; String searchUrl = buildListCommitsUrl(project, since, until); logger.info("getting commits for page " + page); - logger.info("searchUrl=" + searchUrl); Response response = getResponseWithProtection(searchUrl); extractCommits(project, response, commits); @@ -551,36 +613,11 @@ private List getProjectCommits(Project project, String since, String unt logger.info("getting commits for page " + ++page); response = getResponseWithProtection(searchUrl); extractCommits(project, response, commits); - logger.info("done extracting page " + page); } return commits; } - private void extractCommits(Project project, Response response, - List commits) throws IOException { - String jsonResponse = response.getResponseBody(); - JsonElement jsonElement = gson.fromJson(jsonResponse, JsonElement.class); - JsonArray jsonArray = jsonElement.getAsJsonArray(); - - for (JsonElement element : jsonArray) { - Commit commit = gson.fromJson(element, Commit.class); - commit.setProject(project); - - User user = gson.fromJson(element.getAsJsonObject().get("committer"), User.class); - commit.setCommiter(user); - - commit.setMessage(element.getAsJsonObject().get("commit").getAsJsonObject().get("message").getAsString()); - - String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); - commit.setCommitDate(date); - - populateRemainingMetadata(commit); - - commits.add(commit); - } - } - private String buildListCommitsUrl(Project project, String since, String until) { UrlBuilder listCommitsBuilder = builder.uses(GithubAPI.ROOT) @@ -601,35 +638,27 @@ private String buildListCommitsUrl(Project project, String since, return searchUrl; } - private void populateRemainingMetadata(Commit commit) { - JsonElement jsonElement; - String searchUrl = buildCommitSearchUrl(commit); - jsonElement = gson.fromJson(getWithProtection(searchUrl), JsonElement.class); - JsonObject statsObject = jsonElement.getAsJsonObject().get("stats").getAsJsonObject(); - int additions = statsObject.get("additions").getAsInt(); - int deletions = statsObject.get("deletions").getAsInt(); - commit.setAdditionsCount(additions); - commit.setDeletionsCount(deletions); - JsonArray filesArray = jsonElement.getAsJsonObject().get("files").getAsJsonArray(); - List files = new ArrayList<>(); - for (JsonElement fileElement : filesArray) { - CommitFile gitFile = gson.fromJson(fileElement, CommitFile.class); - files.add(gitFile); - } - commit.setFiles(files); - } + private void extractCommits(Project project, Response response, + List commits) throws IOException { + String jsonResponse = response.getResponseBody(); + JsonElement jsonElement = gson.fromJson(jsonResponse, JsonElement.class); + JsonArray jsonArray = jsonElement.getAsJsonArray(); - private String buildCommitSearchUrl(Commit commit) { - String searchUrl; - Project project = commit.getProject(); - searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getUser().getLogin()) - .withSimpleParam("/", project.getName()) - .withParam("/commits") - .withSimpleParam("/", commit.getSha()) - .build(); - return searchUrl; + for (JsonElement element : jsonArray) { + Commit commit = gson.fromJson(element, Commit.class); + commit = getProjectCommit(project, commit.getSha()); + commit.setProject(project); + + User user = gson.fromJson(element.getAsJsonObject().get("committer"), User.class); + commit.setCommiter(user); + + commit.setMessage(element.getAsJsonObject().get("commit").getAsJsonObject().get("message").getAsString()); + + String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); + commit.setCommitDate(date); + + commits.add(commit); + } } /** From ecf2b3f6234150f851cbc3a509d8e87118ea4005 Mon Sep 17 00:00:00 2001 From: Irineu Date: Sun, 9 Feb 2014 11:58:10 -0300 Subject: [PATCH 11/11] Exposing blob URL for a CommitFile --- src/java/main/br/ufpe/cin/groundhog/Commit.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/java/main/br/ufpe/cin/groundhog/Commit.java b/src/java/main/br/ufpe/cin/groundhog/Commit.java index a290299..14642b0 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Commit.java +++ b/src/java/main/br/ufpe/cin/groundhog/Commit.java @@ -220,6 +220,9 @@ public static final class CommitFile { @SerializedName(value="status") private String status; + @SerializedName(value="blob_url") + private String blobUrl; + @SerializedName(value="patch") private String patch; @@ -243,6 +246,10 @@ public String getStatus() { return status; } + public String getBlobUrl() { + return blobUrl; + } + public String getPatch() { return patch; }