Paginate issue discovery to avoid silent data loss at 100+ issues

Add parseLinkHeader() helper to extract the next URL from GitHub API
Link headers. Replace single-fetch issue discovery with a paginated
while loop that follows Link headers until all pages are fetched.

Add unit tests for parseLinkHeader and export from test infrastructure.

Agent-Logs-Url: https://github.com/githubnext/autoloop/sessions/4c3fac44-c844-4240-9fb4-b90b2f2d76bc

Co-authored-by: mrjf <180956+mrjf@users.noreply.github.com>

Author: Copilot

tests/conftest.py

@@ -47,7 +47,7 @@ def _load_workflow_functions():
         f.write(func_source)
         f.write(
             "\n\nmodule.exports = "
-            "{ parseMachineState, parseSchedule, getProgramName, readProgramState };\n"
+            "{ parseMachineState, parseSchedule, getProgramName, readProgramState, parseLinkHeader };\n"
         )
 
     return True
@@ -102,6 +102,7 @@ def _get_program_name_wrapper(pf):
     "parse_machine_state": _parse_machine_state_wrapper,
     "get_program_name": _get_program_name_wrapper,
     "read_program_state": lambda name: _call_js("readProgramState", name),
+    "parse_link_header": lambda header: _call_js("parseLinkHeader", header),
 }
 
 

tests/test_scheduling.py

@@ -20,6 +20,7 @@
 parse_schedule = _funcs["parse_schedule"]
 parse_machine_state = _funcs["parse_machine_state"]
 get_program_name = _funcs["get_program_name"]
+parse_link_header = _funcs["parse_link_header"]
 
 
 # ---------------------------------------------------------------------------
@@ -663,6 +664,35 @@ def test_forced_program_target_metric_fallback_via_frontmatter(self):
         assert target == 0.95
 
 
+# ---------------------------------------------------------------------------
+# parseLinkHeader — extract next-page URL from GitHub API Link header
+# ---------------------------------------------------------------------------
+
+class TestParseLinkHeader:
+    def test_returns_null_for_none(self):
+        assert parse_link_header(None) is None
+
+    def test_returns_null_for_empty_string(self):
+        assert parse_link_header("") is None
+
+    def test_extracts_next_url(self):
+        header = '<https://api.github.com/repos/o/r/issues?page=2&per_page=100>; rel="next", <https://api.github.com/repos/o/r/issues?page=5&per_page=100>; rel="last"'
+        assert parse_link_header(header) == "https://api.github.com/repos/o/r/issues?page=2&per_page=100"
+
+    def test_returns_null_when_no_next(self):
+        header = '<https://api.github.com/repos/o/r/issues?page=1&per_page=100>; rel="prev", <https://api.github.com/repos/o/r/issues?page=5&per_page=100>; rel="last"'
+        assert parse_link_header(header) is None
+
+    def test_next_not_first(self):
+        """next rel is not the first segment."""
+        header = '<https://api.github.com/repos/o/r/issues?page=1&per_page=100>; rel="prev", <https://api.github.com/repos/o/r/issues?page=3&per_page=100>; rel="next", <https://api.github.com/repos/o/r/issues?page=5&per_page=100>; rel="last"'
+        assert parse_link_header(header) == "https://api.github.com/repos/o/r/issues?page=3&per_page=100"
+
+    def test_single_next_segment(self):
+        header = '<https://api.github.com/repos/o/r/issues?page=2&per_page=100>; rel="next"'
+        assert parse_link_header(header) == "https://api.github.com/repos/o/r/issues?page=2&per_page=100"
+
+
 # ---------------------------------------------------------------------------
 # Extraction sanity check — verify conftest.py found the expected functions
 # ---------------------------------------------------------------------------
@@ -677,6 +707,9 @@ def test_parse_machine_state_extracted(self):
     def test_get_program_name_extracted(self):
         assert callable(get_program_name)
 
+    def test_parse_link_header_extracted(self):
+        assert callable(parse_link_header)
+
     def test_read_program_state_extracted(self):
         # read_program_state exists in the workflow but depends on file I/O
         assert "read_program_state" in _funcs

workflows/autoloop.md

@@ -181,6 +181,19 @@ steps:
           }
       }
 
+      // Parse the GitHub API Link header to extract the "next" page URL.
+      // Returns the URL string for the next page, or null if there is none.
+      function parseLinkHeader(header) {
+          if (!header) return null;
+          var parts = header.split(',');
+          for (var i = 0; i < parts.length; i++) {
+              var section = parts[i].trim();
+              var m = section.match(/^<([^>]+)>;\s*rel="next"$/);
+              if (m) return m[1];
+          }
+          return null;
+      }
+
       // Main execution
       async function main() {
           // Bootstrap: create autoloop programs directory and template if missing
@@ -273,18 +286,23 @@ steps:
               } catch (e) { /* stat failed */ }
           }
 
-          // Scan GitHub issues with the 'autoloop-program' label
+          // Scan GitHub issues with the 'autoloop-program' label (paginated)
           const issueProgramsDir = '/tmp/gh-aw/issue-programs';
           fs.mkdirSync(issueProgramsDir, { recursive: true });
           try {
-              const apiUrl = 'https://api.github.com/repos/' + repo + '/issues?labels=autoloop-program&state=open&per_page=100';
-              const response = await fetch(apiUrl, {
-                  headers: {
-                      'Authorization': 'token ' + githubToken,
-                      'Accept': 'application/vnd.github.v3+json',
-                  },
-              });
-              const issues = await response.json();
+              let nextUrl = 'https://api.github.com/repos/' + repo + '/issues?labels=autoloop-program&state=open&per_page=100';
+              const issues = [];
+              while (nextUrl) {
+                  const response = await fetch(nextUrl, {
+                      headers: {
+                          'Authorization': 'token ' + githubToken,
+                          'Accept': 'application/vnd.github.v3+json',
+                      },
+                  });
+                  const page = await response.json();
+                  issues.push(...page);
+                  nextUrl = parseLinkHeader(response.headers.get('link'));
+              }
               for (const issue of issues) {
                   if (issue.pull_request) continue; // skip PRs
                   const body = issue.body || '';