feat: Multi-Step README Generation Pipeline - Fixes Token Limit Issues (#101)

public/robots.txt

@@ -0,0 +1,9 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://readmegen-ai.vercel.app
+
+# Sitemaps
+Sitemap: https://readmegen-ai.vercel.app/sitemap.xml

public/sitemap-0.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
+<url><loc>https://readmegen-ai.vercel.app</loc><lastmod>2026-04-03T16:44:51.843Z</lastmod><changefreq>weekly</changefreq><priority>1</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/docs</loc><lastmod>2026-04-03T16:44:51.843Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/examples</loc><lastmod>2026-04-03T16:44:51.843Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/features</loc><lastmod>2026-04-03T16:44:51.843Z</lastmod><changefreq>monthly</changefreq><priority>0.8</priority></url>
+<url><loc>https://readmegen-ai.vercel.app/generate</loc><lastmod>2026-04-03T16:44:51.843Z</lastmod><changefreq>weekly</changefreq><priority>0.9</priority></url>
+</urlset>

public/sitemap.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<sitemap><loc>https://readmegen-ai.vercel.app/sitemap-0.xml</loc></sitemap>
+</sitemapindex>

src/app/api/generate/route.ts

@@ -1,40 +1,36 @@
-import { NextResponse } from "next/server";
-import { getGeminiModel } from "@/lib/gemini";
-import { getRepoData, getRepoContents } from "@/lib/octokit";
-import { SUPPORTED_LANGUAGES } from "@/constants/languages";
+import { NextRequest, NextResponse } from "next/server";
+import { MultiStepReadmeGenerator } from "@/lib/multi-step-readme-generator";
 
 export const dynamic = "force-dynamic";
 
 /**
- * AI README Generation Endpoint
- * Optimized for data accuracy, clean prompt interpolation, and multi-language support.
+ * Enhanced Multi-Step README Generation Endpoint
  *
- * @param {Request} req - The incoming Next.js/standard Web API Request object containing the repo URL and optional language.
- * @returns {Promise<NextResponse>} A JSON response containing the generated Markdown or an error message.
+ * This endpoint uses a sophisticated multi-step approach to generate READMEs:
+ * 1. Repository Analysis - Smart analysis with token-conscious filtering
+ * 2. Section Planning - Dynamic sections based on project type
+ * 3. Section Generation - Individual section generation within token limits
+ * 4. Assembly & Validation - Retry logic and fallback mechanisms
+ *
+ * Fixes token limit issues from issue #101 by generating sections individually.
  */
-export async function POST(req: Request) {
-  let rawUrl: string;
-  let language: string;
+export async function POST(request: NextRequest) {
   try {
-    const body = await req.json();
-    rawUrl = body.url;
-    language = body.language || "English";
-  } catch {
-    return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 });
-  }
+    const body = await request.json();
+    const { url: githubUrl } = body;
 
-  try {
-    const trimmedUrl = rawUrl?.trim();
-    if (!trimmedUrl) {
+    // Validate required fields
+    if (!githubUrl) {
       return NextResponse.json(
         { error: "GitHub URL is required" },
         { status: 400 },
       );
     }
 
+    // Validate GitHub URL format
     let parsedUrl: URL;
     try {
-      parsedUrl = new URL(trimmedUrl);
+      parsedUrl = new URL(githubUrl.trim());
     } catch {
       return NextResponse.json(
         { error: "Please provide a valid URL" },
@@ -63,107 +59,78 @@ export async function POST(req: Request) {
       );
     }
 
-    const [repoInfo, repoContents] = await Promise.all([
-      getRepoData(owner, repo),
-      getRepoContents(owner, repo),
-    ]);
-
-    const files = Array.isArray(repoContents)
-      ? repoContents.map((f: { name: string }) => f.name)
-      : [];
-    const fileListString =
-      files.length > 0 ? files.join(", ") : "Standard repository structure";
-
-    // Tech Stack detection logic
-    const hasNode = files.includes("package.json");
-    const hasPython =
-      files.includes("requirements.txt") || files.includes("setup.py");
-    const hasDocker =
-      files.includes("Dockerfile") || files.includes("docker-compose.yml");
-
-    // Fix: Cleanly joined Tech Stack labels
-    const stackLabels =
-      [
-        hasNode && "Node.js Environment",
-        hasPython && "Python Environment",
-        hasDocker && "Containerized",
-      ]
-        .filter(Boolean)
-        .join(", ") || "Generic Software Environment";
-
-    // Fix: Dynamic License detection
-    const licenseName =
-      repoInfo?.license?.name ||
-      repoInfo?.license?.spdx_id ||
-      "the repository's license file";
-
-    const model = getGeminiModel();
-
-    // Fix: Prompt updated with neutral fallbacks and dynamic license
-    const prompt = `
-**Role**: You are a Principal Solutions Architect and World-Class Technical Writer. 
-**Task**: Generate a professional, high-conversion README.md for the GitHub repository: "${repo}" in the following language: **${language}**.
-
----
-### 1. PROJECT CONTEXT (VERIFIED DATA)
-- **Project Name**: ${repo}
-- **Description**: ${repoInfo?.description || "No description provided."}
-- **Primary Language**: ${repoInfo?.language || "Language unknown"}
-- **Detected Root Files**: ${fileListString}
-- **Tech Stack Context**: ${stackLabels}
-
----
-### 2. STRICT README STRUCTURE REQUIREMENTS
-
-1. **Visual Header**:
-   - Center-aligned H1 with project name.
-   - A compelling 1-sentence tagline describing the **Value Proposition**.
-   - A centered row of Shields.io badges (Build, License, PRs Welcome, Stars).
-
-2. **The Strategic "Why" (Overview)**:
-   - **The Problem**: Use a blockquote to describe the real-world pain point this project solves.
-   - **The Solution**: Explain how this project provides a superior outcome for the user.
-
-3. **Key Features**:
-   - Minimum 5 features. Use emojis and focus on **User Benefits**.
-
-4. **Technical Architecture**:
-   - Provide a table of the tech stack: | Technology | Purpose | Key Benefit |.
-   - Create a tree-style directory structure code block using 📁 for folders and 📄 for files based on the file manifest provided.
-
-5. **Operational Setup**:
-   - **Prerequisites**: List required runtimes.
-   - **Installation**: Provide step-by-step terminal commands. 
-     ${hasNode ? "- Use npm/yarn/pnpm since package.json was detected." : ""}
-     ${hasPython ? "- Use pip/venv since Python markers were detected." : ""}
-   - **Environment**: If any .env or config files are in the manifest, include a configuration section.
-
-6. **Community & Governance**:
-   - Professional "Contributing" section (Fork -> Branch -> PR).
-   - Detailed "License" section: Reference ${licenseName} and provide a summary of permissions.
-
----
-### 3. TONE & STYLE
-- **Tone**: Authoritative, polished, and developer-centric.
-- **Visuals**: Extensive use of Markdown formatting.
-- **Constraint**: Return ONLY the raw Markdown. No conversational filler.
-    `;
-
-    const result = await model.generateContent(prompt);
-    const response = await result.response;
-    const markdown = response.text().trim();
-    const cleanMarkdown = markdown
-      .replace(/^```(markdown|md)?\n/, "")
-      .replace(/\n```$/, "");
+    // Initialize the multi-step generator with enhanced configuration
+    const generator = new MultiStepReadmeGenerator(
+      process.env.GEMINI_API_KEY!,
+      process.env.GITHUB_TOKEN, // Optional GitHub token for higher rate limits
+      {
+        maxRetries: 3,
+        maxTokensPerSection: 800, // Smaller token limit per section
+        temperature: 0.7,
+        concurrentSections: 3, // Generate multiple sections in parallel
+        enableContinuation: true, // Enable automatic continuation for truncated content
+      },
+    );
 
-    return NextResponse.json({ markdown: cleanMarkdown });
-  } catch (error: unknown) {
-    const message =
-      error instanceof Error ? error.message : "Internal Server Error";
-    console.error("README Generation Failed:", message);
+    // Generate README with detailed tracking
+    const startTime = Date.now();
+    console.log("Starting multi-step README generation for", githubUrl);
+
+    const result = await generator.generateReadme(githubUrl);
+    const endTime = Date.now();
+
+    // Log generation statistics for monitoring
+    console.log("README generation completed for", githubUrl, {
+      success: result.success,
+      sectionsGenerated: result.stats.sectionsGenerated,
+      sectionsTotal: result.stats.sectionsTotal,
+      tokensUsed: result.stats.tokensUsed,
+      timeElapsed: endTime - startTime,
+      errors: result.errors.length,
+    });
+
+    if (!result.success) {
+      console.error("README generation failed:", result.errors);
+      return NextResponse.json(
+        {
+          error: "Failed to generate README using multi-step pipeline",
+          details: result.errors,
+          stats: result.stats,
+        },
+        { status: 500 },
+      );
+    }
 
+    // Return successful result with enhanced metadata
+    return NextResponse.json({
+      success: true,
+      markdown: result.readme, // Keep 'markdown' key for compatibility with existing frontend
+      stats: {
+        sectionsGenerated: result.stats.sectionsGenerated,
+        sectionsTotal: result.stats.sectionsTotal,
+        tokensUsed: result.stats.tokensUsed,
+        timeElapsed: result.stats.timeElapsed,
+        generationMethod: "multi-step", // Indicate the method used
+      },
+      metadata: {
+        name: result.metadata?.name,
+        description: result.metadata?.description,
+        language: result.metadata?.language,
+        stars: result.metadata?.stars,
+        license: result.metadata?.license,
+        projectType: result.structure?.projectType,
+        techStack: result.structure?.techStack.primary,
+        frameworks: result.structure?.techStack.frameworks,
+      },
+      warnings: result.errors.length > 0 ? result.errors : undefined,
+    });
+  } catch (error) {
+    console.error("Multi-step README generation API error:", error);
     return NextResponse.json(
-      { error: "Failed to generate README. Check your URL and try again." },
+      {
+        error: "Internal server error in multi-step README generation",
+        message: error instanceof Error ? error.message : "Unknown error",
+      },
       { status: 500 },
     );
   }