diff --git a/src/java/main/br/ufpe/cin/groundhog/Commit.java b/src/java/main/br/ufpe/cin/groundhog/Commit.java index bb1303d..14642b0 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Commit.java +++ b/src/java/main/br/ufpe/cin/groundhog/Commit.java @@ -1,6 +1,7 @@ package br.ufpe.cin.groundhog; import java.util.Date; +import java.util.List; import br.ufpe.cin.groundhog.util.Dates; @@ -32,9 +33,11 @@ public class Commit extends GitHubEntity { private Date commitDate; - private int additionsCount; - - private int deletionsCount; + private CommitStats stats; + + private List files; + + private List parents; public Commit(String sha, Project project) { this.sha = sha; @@ -103,46 +106,55 @@ public void setCommitDate(String date) { } /** - * Informs the sum of added lines among the files committed - * @param deletionsCount + * Gives the abbreviated SHA of the {@link Commit} object + * @return a {@link String} object */ - public int getAdditionsCount() { - return this.additionsCount; + public String getabbrevSHA() { + return this.sha.substring(0, 7); } - public void setAdditionsCount(int additionsCount) { - this.additionsCount = additionsCount; + public List getFiles() { + return this.files; } - /** - * Informs the sum of deleted lines among the files committed - * @param deletionsCount - */ - public int getDeletionsCount() { - return this.deletionsCount; + public CommitStats getStats() { + return this.stats; } - public void setDeletionsCount(int deletionsCount) { - this.deletionsCount = deletionsCount; + public List getParents() { + return this.parents; } - - /** - * Gives the abbreviated SHA of the {@link Commit} object - * @return a {@link String} object - */ - public String getabbrevSHA() { - return this.sha.substring(0, 7); + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((sha == null) ? 0 : sha.hashCode()); + return result; } - + /** * Two {@link Commit} objects are considered equal if and only if both have the same SHA hash * @param commit * @return */ - public boolean equals(Commit commit) { - return this.sha == commit.sha; + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Commit other = (Commit) obj; + if (sha == null) { + if (other.sha != null) + return false; + } else if (!sha.equals(other.sha)) + return false; + return true; } - + @Override public String toString() { return "Commit [" + (sha != null ? "sha=" + sha + ", " : "") @@ -159,4 +171,115 @@ public String getURL() { this.getProject().getName(), this.sha); } -} \ No newline at end of file + + /** + * The commit changes stats, i.e., number of additions, deletions and total + * @author Irineu + * + */ + public static final class CommitStats { + private int additions; + + private int deletions; + + private int total; + + public int getAdditions() { + return additions; + } + + public int getDeletions() { + return deletions; + } + + public int getTotal() { + return total; + } + + @Override + public String toString() { + return "CommitStats [additions=" + additions + ", deletions=" + + deletions + ", total=" + total + "]"; + } + + } + + public static final class CommitFile { + @SerializedName(value="filename") + private String fileName; + + @SerializedName(value="additions") + private int additionsCount; + + @SerializedName(value="deletions") + private int deletionsCount; + + @SerializedName(value="changes") + private int changesCount; + + @SerializedName(value="status") + private String status; + + @SerializedName(value="blob_url") + private String blobUrl; + + @SerializedName(value="patch") + private String patch; + + public String getFileName() { + return fileName; + } + + public int getAdditionsCount() { + return additionsCount; + } + + public int getDeletionsCount() { + return deletionsCount; + } + + public int getChangesCount() { + return changesCount; + } + + public String getStatus() { + return status; + } + + public String getBlobUrl() { + return blobUrl; + } + + public String getPatch() { + return patch; + } + + @Override + public String toString() { + return "CommitFile [fileName=" + fileName + ", additionsCount=" + + additionsCount + ", deletionsCount=" + deletionsCount + + ", changesCount=" + changesCount + ", status=" + status + + ", patch=" + patch + "]"; + } + } + + public static class CommitParent { + + private String url; + + private String sha; + + public String getUrl() { + return url; + } + + public String getSha() { + return sha; + } + + @Override + public String toString() { + return "CommitParent [url=" + url + ", sha=" + sha + "]"; + } + } +} diff --git a/src/java/main/br/ufpe/cin/groundhog/Issue.java b/src/java/main/br/ufpe/cin/groundhog/Issue.java index bce06e2..b41813d 100644 --- a/src/java/main/br/ufpe/cin/groundhog/Issue.java +++ b/src/java/main/br/ufpe/cin/groundhog/Issue.java @@ -1,300 +1,300 @@ -package br.ufpe.cin.groundhog; - -import java.util.Date; -import java.util.List; - -import org.mongodb.morphia.annotations.Entity; -import org.mongodb.morphia.annotations.Indexed; -import org.mongodb.morphia.annotations.Reference; - -import com.google.gson.annotations.SerializedName; - -/** - * Represents an Issue object in Groundhog - * @author Rodrigo Alves - */ -@Entity("issues") -public class Issue extends GitHubEntity { - @Indexed(unique=true, dropDups=true) - @SerializedName("id") - private int id; - - @SerializedName("number") - private int number; - - @SerializedName("comments") - private int commentsCount; - - @Reference private Project project; - - @SerializedName("pull_request") - private transient PullRequest pullRequest; - - private List labels; - - @Reference private Milestone milestone; - - @SerializedName("title") - private String title; - - @SerializedName("body") - private String body; - - @SerializedName("state") - private String state; - - @SerializedName("assignee") - private User assignee; - - @SerializedName("closed_by") - private User closedBy; - - @SerializedName("created_at") - private Date createdAt; - - @SerializedName("updated_at") - private Date updatedAt; - - @SerializedName("closed_at") - private Date closedAt; - - public Issue(Project project, int number, String state) { - this.number = number; - this.project = project; - this.state = state; - } - - public Issue(Project project, int number, String state, String title) { - this(project, number, state); - this.title = title; - } - - /** - * Returns the ID of the Issue on GitHub. This ID is unique for every Issue on GitHub, which means that - * no two (or more) Issues on GitHub may have the same ID - * @return - */ - public int getId() { - return this.id; - } - - public void setId(int id) { - this.id = id; - } - - /** - * Returns the number of the Issue on its Project. This number is unique within the project to which - * the Issues belong. Thus, the same project may not have two Issues with the same number but two (or more) - * different projects on GitHub may have Issues with the same number. - * @return - */ - public int getNumber() { - return this.number; - } - - public void setNumber(int number) { - this.number = number; - } - - /** - * Informs the number of comments on the Issue - * @return - */ - public int getCommentsCount() { - return this.commentsCount; - } - - public void setCommentsCount(int commentsCount) { - this.commentsCount = commentsCount; - } - - /** - * Informs the Project object to which the Issue belongs - * No Issue exists without a project - * @return - */ - public Project getProject() { - return this.project; - } - - public void setProject(Project project) { - this.project = project; - } - - /** - * Informs the PullRequest related to the Issue. Not all Issues are Pull Request Issues. - * If this method return nulls then it means the Issue is not a PullRequest Issue - * @return - */ - public PullRequest getPullRequest() { - return this.pullRequest; - } - - public void setPullRequest(PullRequest pullRequest) { - this.pullRequest = pullRequest; - } - - public List getLabels() { - return this.labels; - } - - public void setLabels(List labels) { - this.labels = labels; - } - - public Milestone getMilestone() { - return this.milestone; - } - - public void setMilestone(Milestone milestone) { - this.milestone = milestone; - } - - /** - * Returns the title of the Issue - * @return - */ - public String getTitle() { - return this.title; - } - - public void setTitle(String title) { - this.title = title; - } - - /** - * Returns the Markdown-syntax-based description of the Issue - * @return - */ - public String getBody() { - return this.body; - } - - public void setBody(String body) { - this.body = body; - } - - /** - * Returns the current state of the Issue. Possible values are "open" and "closed" - * @return - */ - public String getState() { - return this.state; - } - - public void setState(String state) { - this.state = state; - } - - /** - * Returns the User assigned to that Issue. - * An Issue on GitHub may or may not have an assignee - * @return - */ - public User getAssignee() { - return this.assignee; - } - - public void setAssignee(User assignee) { - this.assignee = assignee; - } - - /** - * Informs the User who closed the Issue - * Every Issue gets closed by someone, so if this value is null - * then the Issue is currently open - * @return - */ - public User getClosedBy() { - return this.closedBy; - } - - public void setClosedBy(User closedBy) { - this.closedBy = closedBy; - } - - /** - * Informs the creation date of the Issue on GitHub - * @return - */ - public Date getCreatedAt() { - return this.createdAt; - } - - public void setCreatedAt(Date createdAt) { - this.createdAt = createdAt; - } - - /** - * Informs the date when the last modification was made upon the issue - * @return - */ - public Date getUpdatedAt() { - return this.updatedAt; - } - - public void setUpdatedAt(Date updatedAt) { - this.updatedAt = updatedAt; - } - - /** - * Informs the date when the Issue was closed - * If the value is null it means the issue is open - * @return - */ - public Date getClosedAt() { - return this.closedAt; - } - - public void setClosedAt(Date closedAt) { - this.closedAt = closedAt; - } - - /** - * Returns true if the Issue is open. Returns false otherwise - * @return - */ - public boolean isOpen() { - return this.getState() == "closed" ? true : false; - } - - /** - * Returns true if the Issue is a Pull Request Issue. Returns false otherwise - * @return - */ - public boolean isPullRequest() { - return this.getPullRequest() != null ? true : false; - } - - /** - * Two {@link Issue} objects are considered equal when they have the same GitHub API ID, - * the same number and the same {@link Project}. - * @param issue - * @return - */ - public boolean equals(Issue issue) { - return this.id == issue.id && this.number == issue.number && this.project.equals(issue.getProject()); - } - - public String getURL() { - return String.format("https://api.github.com/repos/%s/%s/issues/%d", - this.getProject().getOwner().getLogin(), this.getProject().getName(), this.getNumber()); - } - - @Override - public String toString() { - String stringReturn = "Issue Number = " + this.number; - - if (this.title != null) { - stringReturn += ", title: " + this.title; - } - - String url = this.getURL(); // This class doesn't contains a variable referring a URL, so we create one local - - if (url != null) { - stringReturn += ", URL = " + url; - } - - return stringReturn; - } +package br.ufpe.cin.groundhog; + +import java.util.Date; + +import org.mongodb.morphia.annotations.Entity; +import org.mongodb.morphia.annotations.Indexed; +import org.mongodb.morphia.annotations.Reference; + +import com.google.gson.annotations.SerializedName; + +/** + * Represents an Issue object in Groundhog + * @author Rodrigo Alves + */ +@Entity("issues") +public class Issue extends GitHubEntity { + @Indexed(unique=true, dropDups=true) + @SerializedName("id") + private int id; + + @SerializedName("number") + private int number; + + @SerializedName("comments") + private int commentsCount; + + @Reference private Project project; + + @SerializedName("pull_request") + private transient PullRequest pullRequest; + + @SerializedName("labels") + private IssueLabel[] labels; + + @Reference private Milestone milestone; + + @SerializedName("title") + private String title; + + @SerializedName("body") + private String body; + + @SerializedName("state") + private String state; + + @SerializedName("assignee") + private User assignee; + + @SerializedName("closed_by") + private User closedBy; + + @SerializedName("created_at") + private Date createdAt; + + @SerializedName("updated_at") + private Date updatedAt; + + @SerializedName("closed_at") + private Date closedAt; + + public Issue(Project project, int number, String state) { + this.number = number; + this.project = project; + this.state = state; + } + + public Issue(Project project, int number, String state, String title) { + this(project, number, state); + this.title = title; + } + + /** + * Returns the ID of the Issue on GitHub. This ID is unique for every Issue on GitHub, which means that + * no two (or more) Issues on GitHub may have the same ID + * @return + */ + public int getId() { + return this.id; + } + + public void setId(int id) { + this.id = id; + } + + /** + * Returns the number of the Issue on its Project. This number is unique within the project to which + * the Issues belong. Thus, the same project may not have two Issues with the same number but two (or more) + * different projects on GitHub may have Issues with the same number. + * @return + */ + public int getNumber() { + return this.number; + } + + public void setNumber(int number) { + this.number = number; + } + + /** + * Informs the number of comments on the Issue + * @return + */ + public int getCommentsCount() { + return this.commentsCount; + } + + public void setCommentsCount(int commentsCount) { + this.commentsCount = commentsCount; + } + + /** + * Informs the Project object to which the Issue belongs + * No Issue exists without a project + * @return + */ + public Project getProject() { + return this.project; + } + + public void setProject(Project project) { + this.project = project; + } + + /** + * Informs the PullRequest related to the Issue. Not all Issues are Pull Request Issues. + * If this method return nulls then it means the Issue is not a PullRequest Issue + * @return + */ + public PullRequest getPullRequest() { + return this.pullRequest; + } + + public void setPullRequest(PullRequest pullRequest) { + this.pullRequest = pullRequest; + } + + public IssueLabel[] getLabels() { + return this.labels; + } + + public void setLabels(IssueLabel[] labels) { + this.labels = labels; + } + + public Milestone getMilestone() { + return this.milestone; + } + + public void setMilestone(Milestone milestone) { + this.milestone = milestone; + } + + /** + * Returns the title of the Issue + * @return + */ + public String getTitle() { + return this.title; + } + + public void setTitle(String title) { + this.title = title; + } + + /** + * Returns the Markdown-syntax-based description of the Issue + * @return + */ + public String getBody() { + return this.body; + } + + public void setBody(String body) { + this.body = body; + } + + /** + * Returns the current state of the Issue. Possible values are "open" and "closed" + * @return + */ + public String getState() { + return this.state; + } + + public void setState(String state) { + this.state = state; + } + + /** + * Returns the User assigned to that Issue. + * An Issue on GitHub may or may not have an assignee + * @return + */ + public User getAssignee() { + return this.assignee; + } + + public void setAssignee(User assignee) { + this.assignee = assignee; + } + + /** + * Informs the User who closed the Issue + * Every Issue gets closed by someone, so if this value is null + * then the Issue is currently open + * @return + */ + public User getClosedBy() { + return this.closedBy; + } + + public void setClosedBy(User closedBy) { + this.closedBy = closedBy; + } + + /** + * Informs the creation date of the Issue on GitHub + * @return + */ + public Date getCreatedAt() { + return this.createdAt; + } + + public void setCreatedAt(Date createdAt) { + this.createdAt = createdAt; + } + + /** + * Informs the date when the last modification was made upon the issue + * @return + */ + public Date getUpdatedAt() { + return this.updatedAt; + } + + public void setUpdatedAt(Date updatedAt) { + this.updatedAt = updatedAt; + } + + /** + * Informs the date when the Issue was closed + * If the value is null it means the issue is open + * @return + */ + public Date getClosedAt() { + return this.closedAt; + } + + public void setClosedAt(Date closedAt) { + this.closedAt = closedAt; + } + + /** + * Returns true if the Issue is open. Returns false otherwise + * @return + */ + public boolean isOpen() { + return this.getState() == "closed" ? true : false; + } + + /** + * Returns true if the Issue is a Pull Request Issue. Returns false otherwise + * @return + */ + public boolean isPullRequest() { + return this.getPullRequest() != null ? true : false; + } + + /** + * Two {@link Issue} objects are considered equal when they have the same GitHub API ID, + * the same number and the same {@link Project}. + * @param issue + * @return + */ + public boolean equals(Issue issue) { + return this.id == issue.id && this.number == issue.number && this.project.equals(issue.getProject()); + } + + public String getURL() { + return String.format("https://api.github.com/repos/%s/%s/issues/%d", + this.getProject().getOwner().getLogin(), this.getProject().getName(), this.getNumber()); + } + + @Override + public String toString() { + String stringReturn = "Issue Number = " + this.number; + + if (this.title != null) { + stringReturn += ", title: " + this.title; + } + + String url = this.getURL(); // This class doesn't contains a variable referring a URL, so we create one local + + if (url != null) { + stringReturn += ", URL = " + url; + } + + return stringReturn; + } } \ No newline at end of file diff --git a/src/java/main/br/ufpe/cin/groundhog/http/Requests.java b/src/java/main/br/ufpe/cin/groundhog/http/Requests.java index 105d6c9..e22a542 100644 --- a/src/java/main/br/ufpe/cin/groundhog/http/Requests.java +++ b/src/java/main/br/ufpe/cin/groundhog/http/Requests.java @@ -8,6 +8,7 @@ import com.ning.http.client.ListenableFuture; import com.ning.http.client.Request; import com.ning.http.client.RequestBuilder; +import com.ning.http.client.Response; /** * Utility class to perform asynchronous http requests @@ -35,6 +36,19 @@ public String get(String urlStr) { throw new HttpException(e); } } + + /** + * Gets the {@link Response} object for a given URL request + * @param urlStr the URL for the request to be performed + * @return the {@link Response} object for this request + */ + public Response getResponse(String urlStr) { + try { + return this.httpClient.prepareGet(urlStr).execute().get(); + } catch (Exception e) { + throw new HttpException(e); + } + } /** * Gets the response body of the given URL using the preview flag in the request header diff --git a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java index 4fe782e..ec5306b 100644 --- a/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java +++ b/src/java/main/br/ufpe/cin/groundhog/search/SearchGitHub.java @@ -1,10 +1,18 @@ package br.ufpe.cin.groundhog.search; import static br.ufpe.cin.groundhog.http.URLsDecoder.encodeURL; +import static com.google.common.base.Preconditions.checkNotNull; +import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,12 +35,10 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; import com.google.inject.Guice; import com.google.inject.Inject; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map.Entry; +import com.ning.http.client.Response; /** * Performs the project search on GitHub, via its official JSON API @@ -41,8 +47,20 @@ * @since 0.0.1 */ public class SearchGitHub implements ForgeSearch { + private static Logger logger = LoggerFactory.getLogger(SearchGitHub.class); - + + /* + * Default number of items per page as defined in http://developer.github.com/v3/#pagination + */ + @SuppressWarnings("unused") + private static final int DEFAULT_PAGINATION_LIMIT = 30; + + /* + * Pattern for a Github Link header + */ + private static final Pattern LINK_PATTERN = Pattern.compile("\\<(?\\S+)\\>\\; rel=\\\"(?\\S+)\\\""); + public static int INFINITY = -1; private final Gson gson; @@ -251,7 +269,7 @@ public List getProjects(String term, String username, int page) throws for (JsonElement j: jsonArray) { Project p = gson.fromJson(j, Project.class); JsonObject jsonObj = j.getAsJsonObject(); - + p.setSCM(SCM.GIT); p.setScmURL(String.format("git@github.com:%s/%s.git", username, p.getName())); p.setSourceCodeURL(jsonObj.get("url").getAsString()); @@ -302,34 +320,93 @@ public List getProjectLanguages(Project project) { } /** - * Fetches all the Issues of the given {@link Project} from the GitHub API + * Fetches all the Issues (open and closed) of the given {@link Project} from the GitHub API * * @param project the @{link Project} of which the Issues are about * @return a {@link List} of {@link Issues} objects */ public List getAllProjectIssues(Project project) { + //XXX: Instead fetching ALL issues at once we could return a lazy collection that would + // do it on the fly + try { + return getProjectIssuesInner(project); + } catch (JsonSyntaxException | IOException e) { + throw new SearchException(e); + } + } + private List getProjectIssuesInner(Project project) throws JsonSyntaxException, IOException { logger.info("Searching project issues metadata"); - - String searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getSourceCodeURL().split("/")[3]) - .withSimpleParam("/", project.getName()) - .withParam("/issues") - .build(); - - String jsonString = requests.get(searchUrl); - JsonArray jsonArray = gson.fromJson(jsonString, JsonElement.class).getAsJsonArray(); + List issues = getAllProjectIssuesForState(project, "open"); + issues.addAll(getAllProjectIssuesForState(project, "closed")); + return issues; + } + private List getAllProjectIssuesForState(Project project, + String state) throws IOException { List issues = new ArrayList(); - + String searchUrl = buildListIssuesUrl(project.getUser().getLogin(), project.getName(), 1, state); + Response response = getResponseWithProtection(searchUrl); + String jsonResponse = response.getResponseBody(); + extractIssues(project, jsonResponse, issues); + while((searchUrl = getNextUrl(response)) != null){ + response = getResponseWithProtection(searchUrl); + jsonResponse = response.getResponseBody(); + extractIssues(project, jsonResponse, issues); + } + return issues; + } + + private void extractIssues(Project project, String jsonResponse, + List issues) { + JsonArray jsonArray = gson.fromJson(jsonResponse, JsonElement.class).getAsJsonArray(); for (JsonElement element : jsonArray) { Issue issue = gson.fromJson(element, Issue.class); issue.setProject(project); issues.add(issue); } + } - return issues; + private String buildListIssuesUrl(String user, String project, int page, String state){ + String searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("repos") + .withSimpleParam("/", user) + .withSimpleParam("/", project) + .withSimpleParam("/", "issues") + .withParam("state", state) + .withParam("page", "" + page) + .build(); + return searchUrl; + } + + private String getNextUrl(Response response) { + String nextLink = null; + String linkHeader = response.getHeader("Link"); + if(linkHeader != null){ + Map relToLinks = getLinks(linkHeader); + if(relToLinks.containsKey("next")){ + nextLink = relToLinks.get("next"); + } + } + return nextLink; + } + + /* + * Extract the links out of a Link header + */ + private Map getLinks(String linkHeader) { + Map relToLinks = new HashMap<>(); + Matcher linkMatcher = LINK_PATTERN.matcher(linkHeader); + while(linkMatcher.find()){ + String rel = linkMatcher.group("rel"); + String link = linkMatcher.group("link"); + if(!relToLinks.containsKey(rel)) relToLinks.put(rel, link); + else { + logger.warn("Duplicate rel, previous link " + relToLinks.get(rel) + + " , new link " + link); + } + } + return relToLinks; } /** @@ -404,41 +481,109 @@ public int getNumberProjectTags(Project project) { } /** - * Fetches all the Commits of the given {@link Project} from the GitHub API - * @param project the @{link Project} to which the commits belong - * @return a {@link List} of {@link Commit} objects + * Returns the commit with the given SHA sha for project + * @param project + * @param sha + * @return */ - public List getAllProjectCommits(Project project) { - logger.info("Searching project commits metadata"); - - String searchUrl = builder.uses(GithubAPI.ROOT) - .withParam("repos") - .withSimpleParam("/", project.getSourceCodeURL().split("/")[3]) - .withSimpleParam("/", project.getName()) - .withParam("/commits") - .build(); - - System.out.println(searchUrl); + public Commit getProjectCommit(Project project, String sha) { + checkNotNull(project); + checkNotNull(sha); + Commit commit = null; + String searchUrl = buildCommitSearchUrl(project, sha); + String responseBody = getWithProtection(searchUrl); + commit = gson.fromJson(responseBody, Commit.class); + return commit; + } - JsonElement jsonElement = gson.fromJson(requests.get(searchUrl), JsonElement.class); - JsonArray jsonArray = jsonElement.getAsJsonArray(); + private String buildCommitSearchUrl(Project project, String sha) { + String searchUrl; + searchUrl = builder.uses(GithubAPI.ROOT) + .withParam("repos") + .withSimpleParam("/", project.getUser().getLogin()) + .withSimpleParam("/", project.getName()) + .withParam("/commits") + .withSimpleParam("/", sha) + .build(); + return searchUrl; + } - List commits = new ArrayList<>(); - for (JsonElement element : jsonArray) { - Commit commit = gson.fromJson(element, Commit.class); - commit.setProject(project); - - User user = gson.fromJson(element.getAsJsonObject().get("committer"), User.class); - commit.setCommiter(user); - - commit.setMessage(element.getAsJsonObject().get("commit").getAsJsonObject().get("message").getAsString()); + /** + * Returns the commit pointed by refSpec for Project project. + * @param project + * @return + */ + public Commit getProjectCommitByRef(Project project, String refSpec){ + checkNotNull(project); + checkNotNull(refSpec); + Commit head = null; + String refSHA = getRefSHA(project, refSpec); + if(refSHA != null) { + head = getProjectCommit(project, refSHA); + } + return head; + } - String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); - commit.setCommitDate(date); - commits.add(commit); + private String getRefSHA(Project project, String refName) { + String sha = null; + String searchUrl = buildRefSearchUrl(project, refName); + String responseBody = getWithProtection(searchUrl); + JsonElement ref = gson.fromJson(responseBody, JsonElement.class); + JsonElement object = ref.getAsJsonObject().get("object"); + sha = object.getAsJsonObject().get("sha").getAsString(); + return sha; + } + + private String buildRefSearchUrl(Project project, String refName) { + return builder.uses(GithubAPI.ROOT). + withParam("repos"). + withSimpleParam("/", project.getUser().getLogin()). + withSimpleParam("/", project.getName()). + withSimpleParam("/", "git"). + withSimpleParam("/", "refs"). + withSimpleParam("/", refName).build(); + } + + /** + * Fetches all the Commits of the given {@link Project} from the GitHub API for the specified + * period. If either since or until are null the period is open-ended, if both are + * null then this method behaves exactly like {@link #getAllProjectCommits(Project)} + * @param project project to fetch commits metadata for + * @param since initial date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @param until end date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsByPeriod(Project project, String since, String until){ + checkNotNull(project, "project must not be null"); + logger.info("Searching project commits metadata by period"); + try { + return getProjectCommits(project, since, until); + } catch (IOException e) { + throw new SearchException(e); } + } - return commits; + /** + * Utility method for getAllProjectCommitsByPeriod(project, since, null) + * @param project project to fetch commits metadata for + * @param since initial date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsSince(Project project, String since){ + return getProjectCommitsByPeriod(project, since, null); + } + + /** + * Utility method for getAllProjectCommitsByPeriod(project, null, until) + * @param project project to fetch commits metadata for + * @param until end date of the period in the ISO 8601 format YYYY-MM-DDTHH:MM:SSZ + * @return the list of commits from project project within the specified period, + * or an empty list if no such commits exist + */ + public List getProjectCommitsUntil(Project project, String until){ + return getProjectCommitsByPeriod(project, null, until); } /** @@ -446,35 +591,74 @@ public List getAllProjectCommits(Project project) { * @param project the @{link Project} to which the commits belong * @return a {@link List} of {@link Commit} objects */ - public List getAllProjectCommitsByDate(Project project, String start, String end) { + public List getAllProjectCommits(Project project) { + checkNotNull(project, "project must not be null"); + logger.info("Searching all project commits metadata"); + try { + return getProjectCommits(project, null, null); + } catch (IOException e) { + throw new SearchException(e); + } + } - logger.info("Searching all project commits metadata by date"); + private List getProjectCommits(Project project, String since, String until) throws IOException { + List commits = new ArrayList<>(); + int page = 1; + String searchUrl = buildListCommitsUrl(project, since, until); + logger.info("getting commits for page " + page); + Response response = getResponseWithProtection(searchUrl); + extractCommits(project, response, commits); - String searchUrl = builder.uses(GithubAPI.ROOT) + while((searchUrl = getNextUrl(response)) != null){ + logger.info("getting commits for page " + ++page); + response = getResponseWithProtection(searchUrl); + extractCommits(project, response, commits); + } + + return commits; + } + + private String buildListCommitsUrl(Project project, String since, + String until) { + UrlBuilder listCommitsBuilder = builder.uses(GithubAPI.ROOT) .withParam("repos") .withSimpleParam("/", project.getUser().getLogin()) .withSimpleParam("/", project.getName()) - .withSimpleParam("/", "commits") - .withParam("since", start) - .withParam("until", end) - .build(); + .withSimpleParam("/", "commits"); - String response = getWithProtection(searchUrl); + if(since != null){ + listCommitsBuilder = listCommitsBuilder.withParam("since", since); + } - JsonElement jsonElement = gson.fromJson(response, JsonElement.class); + if(until != null) { + listCommitsBuilder = listCommitsBuilder.withParam("until", until); + } + + String searchUrl = listCommitsBuilder.build(); + return searchUrl; + } + + private void extractCommits(Project project, Response response, + List commits) throws IOException { + String jsonResponse = response.getResponseBody(); + JsonElement jsonElement = gson.fromJson(jsonResponse, JsonElement.class); JsonArray jsonArray = jsonElement.getAsJsonArray(); - List commits = new ArrayList<>(); for (JsonElement element : jsonArray) { Commit commit = gson.fromJson(element, Commit.class); + commit = getProjectCommit(project, commit.getSha()); commit.setProject(project); + + User user = gson.fromJson(element.getAsJsonObject().get("committer"), User.class); + commit.setCommiter(user); + + commit.setMessage(element.getAsJsonObject().get("commit").getAsJsonObject().get("message").getAsString()); String date = element.getAsJsonObject().get("commit").getAsJsonObject().get("author").getAsJsonObject().get("date").getAsString(); commit.setCommitDate(date); + commits.add(commit); } - - return commits; } /** @@ -665,18 +849,34 @@ public List getAllProjectReleases(Project project) { } private String getWithProtection(String url){ - String data = requests.get(url); - - if (data.contains("API rate limit exceeded for")) { - try { - Thread.sleep(1000 * 60 * 60); - data = requests.get(url); + try { + return getResponseWithProtection(url).getResponseBody(); + } catch (IOException e) { + // This should never happen, but let's be safe + throw new SearchException(e); + } + } - } catch (InterruptedException ex) { - ex.printStackTrace(); + private Response getResponseWithProtection(String url){ + Response response = requests.getResponse(url); + String data; + try { + int statusCode = response.getStatusCode(); + + data = response.getResponseBody(); + // 403 == forbidden + if(statusCode == 403 && data != null && data.contains("API rate limit exceeded")) { + try { + logger.info("API rate limit exceeded, waiting for " + (60 * 60) + " seconds"); + Thread.sleep(1000 * 60 * 60); + data = requests.get(url); + } catch (InterruptedException ex) { + ex.printStackTrace(); + } } + } catch (IOException e) { + throw new SearchException(e); } - - return data; + return response; } -} \ No newline at end of file +} diff --git a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java index d9d640c..e1e039d 100644 --- a/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java +++ b/src/java/test/br/ufpe/cin/groundhog/search/SearchGitHubTest.java @@ -7,6 +7,7 @@ import org.junit.Test; import br.ufpe.cin.groundhog.Commit; +import br.ufpe.cin.groundhog.Issue; import br.ufpe.cin.groundhog.Language; import br.ufpe.cin.groundhog.Project; import br.ufpe.cin.groundhog.Release; @@ -118,4 +119,24 @@ public void testGetAllProjectReleases() { Assert.fail(); } } + + // Note that in the future this test may have to be reworked as the number of issues in the + // target project grows + @Test + public void testGetAllProjectIssues(){ + try { + User u = new User("spgroup"); + Project project = new Project(u, "groundhog"); + + List projectIssues = searchGitHub.getAllProjectIssues(project); + + Assert.assertNotNull(projectIssues); + // As of 01-Jan-2014 the number of issues for project spgroup/groundhog is 315 and + // this number can only grow + Assert.assertTrue(projectIssues.size() >= 315); + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(); + } + } } \ No newline at end of file