diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index dad24232af..c3dcb3ce56 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -243,10 +243,33 @@ def github_api_get_request(request_f, github_user=None, token=None, **kwargs): url = request_f(RestClient(GITHUB_API_URL, username=github_user, token=token)) try: - status, data = url.get(**kwargs) + status, headers = url.head(**kwargs) except socket.gaierror as err: - _log.warning("Error occurred while performing get request: %s", err) - status, data = 0, None + _log.warning("Error occurred while performing head request: %s", err) + status, headers = 0, None + + pages = [1] + if headers: + links = headers.getheader('Link') + if links: + match = re.search(r"page=(\d+)>; rel=\"last\"", links) + if match: + last = int(match.groups()[0]) + pages = range(1, last+1) + + data = [] + for page in pages: + kwargs['page'] = page + try: + status, page_data = url.get(**kwargs) + except socket.gaierror as err: + _log.warning("Error occurred while performing get request: %s", err) + status, page_data = 0, None + break + if isinstance(page_data, list): + data.extend(page_data) + else: + data = page_data _log.debug("get request result for %s: status: %d, data: %s", url, status, data) return (status, data) diff --git a/test/framework/github.py b/test/framework/github.py index 13fa70264a..53f4901835 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -152,6 +152,11 @@ def test_list_prs(self): output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT) self.assertEqual(expected, output) + # test handling of github pagination + parameters = ('all', 'created', 'asc') + output = gh.list_prs(parameters, per_page=1, github_user=GITHUB_TEST_ACCOUNT) + self.assertTrue(len(output.splitlines()) > 1) + def test_reasons_for_closing(self): """Test reasons_for_closing function.""" if self.skip_github_tests: