Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions deploystream/apps/feature/lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
#-*- coding: utf-8 -*-

from deploystream import app
from deploystream.exceptions import UnknownProviderException
from deploystream.exceptions import (
UnknownProviderException, UnknownFeatureException)
from deploystream.providers.interfaces import (
IBuildInfoProvider, IPlanningProvider, ISourceCodeControlProvider)
from .models import Branch, BuildInfo, Feature
Expand Down Expand Up @@ -46,8 +47,13 @@ def get_feature_info(feature_provider, feature_id, providers):
# First get feature info from the management provider
planning_provider = providers[feature_provider]

feature = Feature(planning_provider,
**planning_provider.get_feature_info(feature_id))
feature_info = planning_provider.get_feature_info(feature_id)

if not feature_info:
raise UnknownFeatureException(feature_id)

feature = Feature(planning_provider, **feature_info)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is something still nagging me with this approach.

When getting feature info from a provider this way, it will make it more difficult to correlate and aggregate data from various sources (e.g. planning info in sprintly, open PRs in github, branches in github/git, jobs in Jenkins...). An alternative approach is passing a Feature object (or a dict if you prefer) around to the various providers in sequence (starting with the planning provider) so that each decorates the feature with additional data it knows about.

For example, I would like to get initial feature info from sprintly, then pass this to the github provider so that it adds information about open PRs and relevant branches (and even mergeability status and build success from the PR), then to Jenkins so that it can add information about known builds to the feature's branches...

Up for hangout discussion.


# Then get any branch info from any source control providers
for provider in providers[ISourceCodeControlProvider]:
for branch_data in provider.get_repo_branches_involved(
Expand Down
5 changes: 4 additions & 1 deletion deploystream/apps/feature/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
from deploystream.apps.feature.lib import get_feature_info, get_all_features
from deploystream.lib.transforms import nativify
from deploystream.decorators import needs_providers
from deploystream.exceptions import UnknownProviderException
from deploystream.exceptions import (
UnknownProviderException, UnknownFeatureException)


def as_json(func):
Expand Down Expand Up @@ -38,4 +39,6 @@ def view_feature(source_id, feature_id, providers):
feature = get_feature_info(source_id, feature_id, providers)
except UnknownProviderException:
abort(404)
except UnknownFeatureException:
abort(404)
return feature
4 changes: 4 additions & 0 deletions deploystream/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,7 @@ def __init__(self, missing_token, *args, **kwargs):

class UnknownProviderException(Exception):
pass


class UnknownFeatureException(Exception):
pass
66 changes: 43 additions & 23 deletions deploystream/providers/github/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import github3
import datetime
import iso8601
import re

import github3
from zope import interface

from deploystream.providers.interfaces import IPlanningProvider
Expand Down Expand Up @@ -70,33 +73,42 @@ def get_features(self, **filters):
repository.name)
if repository.has_issues:
for issue in repository.iter_issues(**filters):
issue_info = transforms.remap(issue.__dict__, FEATURE_MAP)
if issue.pull_request:
issue_type = 'PR'
else:
issue_type = 'story'
issue_info['type'] = issue_type
issue_info['project'] = project
owner = issue_info['assignee']
if owner is None:
issue_info['owner'] = ''
else:
# take only login name from User object
issue_info['owner'] = owner.login
issue_info = self._convert_to_dict(issue, project)
features.append(issue_info)

# sort by putting PRs first, stories second
features = sorted(features, key=lambda f: f['type'] == 'story')

return features

def _convert_to_dict(self, issue, project):
issue_info = transforms.remap(issue.__dict__, FEATURE_MAP)
if issue.pull_request:
issue_type = 'PR'
else:
issue_type = 'story'
issue_info['type'] = issue_type
issue_info['project'] = project
owner = issue_info['assignee']
if owner is None:
issue_info['owner'] = ''
else:
# take only login name from User object
issue_info['owner'] = owner.login
return issue_info

def get_feature_info(self, feature_id):
# Feature ID will need to have org in it.
# For now we'll do a really crude search through the get_features
# results
for feat in self.get_features():
if str(feat['id']) == str(feature_id):
return feat
# Issue with this approach is that we return the first issue with an
# ID across all repos.
# Such are the shortcomings of using Git as a planning provider. To
# get round this we'd need to have the repo in the feature_id, but this
# seems a bad idea from the POV of matching branch names.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup this is a hairy issue. We may want to think of a decent solution to that.

for repository in self.repositories:
project = '{0}/{1}'.format(repository.owner.login,
repository.name)
issue = repository.issue(feature_id)
if issue:
return self._convert_to_dict(issue, project)

@classmethod
def get_oauth_data(self):
Expand Down Expand Up @@ -135,6 +147,7 @@ def get_repo_branches_involved(self, feature_id, hierarchy_regexes):

"""
branch_list = []
two_months_ago = datetime.datetime.now() - datetime.timedelta(60)

for repo in self.repositories:
repo_branches = {}
Expand All @@ -157,9 +170,16 @@ def get_repo_branches_involved(self, feature_id, hierarchy_regexes):
# we haven't already done so and store them in the
# temporary ``repo_branches`` dict
if repo_branches[sha].get('commits') is None:
repo_branches[sha]['commits'] = [
c.sha for c in repo.iter_commits(sha=sha)
]
c_list = []
for commit in repo.iter_commits(sha=sha):
commit_date = commit.commit.committer['date']
commit_date_time = iso8601.parse_date(
commit_date)
if (commit_date_time.replace(tzinfo=None) <
two_months_ago):
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For now I'm using this to stop going back to the beginning of time for every branch that matches the feature id.

Sooner rather than later we're going to need a chat about switching the architecture to being pre-fetched. Are you free at all this week?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the best local caching for repos is going to be local clones as in storyboard, which will make all these more efficient.

Give this a read and see possible solutions, I am thinking deploy keys but there are other options: https://help.github.com/articles/managing-deploy-keys

One can create deploy keys via the API: http://developer.github.com/v3/repos/keys/

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We'll revisit this in the next round of changes following the hangout I guess.

I'm going to set up some planning tool - I think we could do with some stories & tasks to get this going.

break
c_list.append(commit.sha)
repo_branches[sha]['commits'] = c_list
# Check if we're merged in
parent_data = repo_branches[parent]
has_parent = parent_data['sha'] in branch_data['commits']
Expand Down
6 changes: 4 additions & 2 deletions deploystream/providers/jira/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def __init__(self, user, password, url, issue_types=None):
if not issue_types:
issue_types = ['Story', 'Bug']
self.issue_types = issue_types
self.base_url = url
options = {'server': url}
self._conn = JIRA(options, basic_auth=(user, password))

Expand Down Expand Up @@ -78,5 +79,6 @@ def get_feature_info(self, feature_id):

``None`` otherwise
"""
raise NotImplementedError("get_feature_info() not implemented in {0}"
.format(self.__class__.__name__))
jira_feature = self._conn.issue(feature_id)
if jira_feature:
return _transform(jira_feature)
1 change: 1 addition & 0 deletions requirements/runtime.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ zope.interface # define and enforce interfaces
certifi # Module for Mozilla's CA bundle
apitopy # Required to implement the sprint.ly client
jira-python>=0.13 # Required to access JIRA api
iso8601 # Parsing iso8601 datetimes.
21 changes: 15 additions & 6 deletions tests/test_providers/test_github_provider.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import datetime
from mock import Mock, patch
from nose.tools import assert_equal, assert_true

Expand All @@ -7,11 +8,19 @@
from tests import DEFAULT_HIERARCHY_REGEXES


def mock_commit(sha):
mock_comm = Mock(sha=sha)
mock_comm.commit.committer = {'date': datetime.datetime.now().isoformat()}
return mock_comm


def mock_github3(github3):
mock_repo = Mock()
mock_repo.has_issues = True
mock_repo.name = 'repo_1'
mock_repo.iter_commits.return_value = [Mock(sha="CoMmItHaSh-MaStEr")]
mock_repo.iter_commits.return_value = [
mock_commit(sha="CoMmItHaSh-MaStEr")
]

issue1 = {
'title': 'Hello',
Expand All @@ -36,15 +45,15 @@ def mock_github3(github3):

branch1 = {
'name': 'master',
'commit': Mock(sha='CoMmItHaSh-MaStEr'),
'commit': mock_commit(sha='CoMmItHaSh-MaStEr'),
}
branch2 = {
'name': 'story/5/alex',
'commit': Mock(sha='CoMmItHaSh-5'),
'commit': mock_commit(sha='CoMmItHaSh-5'),
}
branch3 = {
'name': 'story/23/alex',
'commit': Mock(sha='CoMmItHaSh-23'),
'commit': mock_commit(sha='CoMmItHaSh-23'),
}
mock_branch1, mock_branch2, mock_branch3 = Mock(), Mock(), Mock()
mock_branch1.__dict__ = branch1
Expand Down Expand Up @@ -82,8 +91,8 @@ def test_implements_expected_interfaces(_):
def test_get_repo_branches_involved(github3):
mock_github3(github3)
github_provider = GithubProvider('token')
branches = github_provider.get_repo_branches_involved("5",
DEFAULT_HIERARCHY_REGEXES)
branches = github_provider.get_repo_branches_involved(
"5", DEFAULT_HIERARCHY_REGEXES)
assert_equal(2, len(branches))
assert_true({
"repository": "repo_1",
Expand Down