From 67409d2204a17ebe338d68a74d4211aad0f1203c Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 29 Jan 2019 15:21:06 +0800 Subject: [PATCH 1/5] handle github api pagination when present --- easybuild/tools/github.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 77af6307c5..1527ac7391 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -243,10 +243,32 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) try: - status, data = url.get(**kwargs) + status, headers = url.head(**kwargs) except socket.gaierror as err: _log.warning("Error occurred while performing get request: %s", err) - status, data = 0, None + status, headers = 0, None + + links = headers.getheader('Link') + + if links: + match = re.search(r"page=(\d+)>; rel=\"last\"", links) + last = int(match.groups()[0]) if match else 1 + data = [] + for page in range(1, last+1): + kwargs['page'] = page + try: + status, page_data = url.get(**kwargs) + except socket.gaierror as err: + _log.warning("Error occurred while performing get request: %s", err) + status, page_data = 0, None + break + data.extend(page_data) + else: + try: + status, data = url.get(**kwargs) + except socket.gaierror as err: + _log.warning("Error occurred while performing get request: %s", err) + status, data = 0, None _log.debug("get request result for %s: status: %d, data: %s", url, status, data) return (status, data) From 68bf52b237f8ea8e2ad78ab49255d21112edb7e1 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Tue, 29 Jan 2019 15:50:52 +0800 Subject: [PATCH 2/5] get link header inside try block --- easybuild/tools/github.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 1527ac7391..4af671eb99 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -244,11 +244,10 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): try: status, headers = url.head(**kwargs) + links = headers.getheader('Link') except socket.gaierror as err: _log.warning("Error occurred while performing get request: %s", err) - status, headers = 0, None - - links = headers.getheader('Link') + status, headers, links = 0, None, None if links: match = re.search(r"page=(\d+)>; rel=\"last\"", links) From 4a5166cf44ce58a2925b34b05a34b5574d6d162b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 21 Feb 2019 10:26:19 +0800 Subject: [PATCH 3/5] refactor handling of github pagination --- easybuild/tools/github.py | 45 +++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4af671eb99..9f621c5609 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -243,31 +243,34 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) try: - status, headers = url.head(**kwargs) - links = headers.getheader('Link') + status, headers = url.head(**kwargs) except socket.gaierror as err: - _log.warning("Error occurred while performing get request: %s", err) - status, headers, links = 0, None, None - - if links: - match = re.search(r"page=(\d+)>; rel=\"last\"", links) - last = int(match.groups()[0]) if match else 1 - data = [] - for page in range(1, last+1): - kwargs['page'] = page - try: - status, page_data = url.get(**kwargs) - except socket.gaierror as err: - _log.warning("Error occurred while performing get request: %s", err) - status, page_data = 0, None - break - data.extend(page_data) - else: + _log.warning("Error occurred while performing head request: %s", err) + status, headers = 0, None + + pages = [1] + if headers: + links = headers.getheader('Link') + if links: + match = re.search(r"page=(\d+)>; rel=\"last\"", links) + if match: + last = int(match.groups()[0]) + pages = range(1, last+1) + + data = [] + for page in pages: + kwargs['page'] = page try: - status, data = url.get(**kwargs) + status, page_data = url.get(**kwargs) except socket.gaierror as err: _log.warning("Error occurred while performing get request: %s", err) - status, data = 0, None + status, page_data = 0, None + break + if isinstance(page_data, list): + data.extend(page_data) + else: + data = page_data + break _log.debug("get request result for %s: status: %d, data: %s", url, status, data) return (status, data) From 5e44ce656129307de8fd5ac846bca2ac3375c218 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 21 Feb 2019 11:35:40 +0800 Subject: [PATCH 4/5] test handling of github pagination --- test/framework/github.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/framework/github.py b/test/framework/github.py index 13fa70264a..53f4901835 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -152,6 +152,11 @@ def test_list_prs(self): output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(expected, output) + # test handling of github pagination + parameters = ('all', 'created', 'asc') + output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT) + self.assertTrue(len(output.splitlines()) > 1) + def test_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: From fcd265fa7d0e917e2e381fda6dfb77552e6e9479 Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 21 Feb 2019 20:36:35 +0800 Subject: [PATCH 5/5] remove unnecessary break --- easybuild/tools/github.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 4b4f5eb068..c3dcb3ce56 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -270,7 +270,6 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): data.extend(page_data) else: data = page_data - break _log.debug("get request result for %s: status: %d, data: %s", url, status, data) return (status, data)