From 957d79191770a1f525dd2521ddc841a03492d1b5 Mon Sep 17 00:00:00 2001 From: Chloe Lin Date: Fri, 20 Feb 2026 14:46:11 +0100 Subject: [PATCH 1/2] increase page size for fetching authored list --- app/user/client-page.tsx | 13 +++++-------- app/user/page.tsx | 21 ++++++++------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/app/user/client-page.tsx b/app/user/client-page.tsx index 0911a1645..999e78882 100644 --- a/app/user/client-page.tsx +++ b/app/user/client-page.tsx @@ -1,11 +1,11 @@ "use client"; -import { useEffect, useState, useMemo } from "react"; import { useSearchParams } from "next/navigation"; -import Breadcrumbs from "@/components/Breadcrumbs"; -import Spinner from "@/components/Spinner"; +import { useEffect, useMemo, useState } from "react"; import { FaUserCircle } from "react-icons/fa"; +import Breadcrumbs from "@/components/Breadcrumbs"; import RuleList from "@/components/rule-list"; +import Spinner from "@/components/Spinner"; import Pagination from "@/components/ui/pagination"; const Tabs = { @@ -15,7 +15,7 @@ const Tabs = { type TabKey = (typeof Tabs)[keyof typeof Tabs]; -export default function UserRulesClientPage({ ruleCount }) { +export default function UserRulesClientPage() { const searchParams = useSearchParams(); const [activeTab, setActiveTab] = useState(Tabs.AUTHORED); const queryStringRulesAuthor = searchParams.get("author") || ""; @@ -30,13 +30,10 @@ export default function UserRulesClientPage({ ruleCount }) { const [authoredRules, setAuthoredRules] = useState([]); const [author, setAuthor] = useState<{ fullName?: string; slug?: string; gitHubUrl?: string }>({}); const [loadingAuthored, setLoadingAuthored] = useState(false); - const [authoredNextCursor, setAuthoredNextCursor] = useState(null); - const [authoredHasNext, setAuthoredHasNext] = useState(false); - const [loadingMoreAuthored, setLoadingMoreAuthored] = useState(false); const [githubError, setGithubError] = useState(null); const [currentPageAuthored, setCurrentPageAuthored] = useState(1); const [itemsPerPageAuthored, setItemsPerPageAuthored] = useState(20); - const FETCH_PAGE_SIZE = 10; + const FETCH_PAGE_SIZE = 100; const resolveAuthor = async (): Promise => { const res = await fetch(`./api/crm/employees?query=${encodeURIComponent(queryStringRulesAuthor)}`); diff --git a/app/user/page.tsx b/app/user/page.tsx index 6192fac94..e035ed337 100644 --- a/app/user/page.tsx +++ b/app/user/page.tsx @@ -1,22 +1,17 @@ -import Layout from '@/components/layout/layout'; -import { Section } from '@/components/layout/section'; -import UserRulesClientPage from './client-page'; import { Suspense } from "react"; -import { fetchRuleCount } from '@/lib/services/rules'; -import { siteUrl } from '@/site-config'; +import { Section } from "@/components/layout/section"; +import { siteUrl } from "@/site-config"; +import UserRulesClientPage from "./client-page"; export const revalidate = 300; export default async function UserRulesPage() { - - const ruleCount = await fetchRuleCount() - return ( -
- - - -
+
+ + + +
); } From 834dfdbe2194eb35074d39240a275c89670270cd Mon Sep 17 00:00:00 2001 From: Chloe Lin Date: Fri, 20 Feb 2026 16:55:20 +0100 Subject: [PATCH 2/2] Add author rules index generation and caching for API optimization --- .github/workflows/build-artifacts.yml | 15 +++ .gitignore | 1 + README.md | 7 +- app/api/tina/rules-by-author/route.ts | 111 +++++++++++++++-- package.json | 1 + scripts/generate-author-rules-map.js | 171 ++++++++++++++++++++++++++ scripts/prepare-content-build.js | 4 + scripts/prepare-content.js | 4 + 8 files changed, 303 insertions(+), 11 deletions(-) create mode 100644 scripts/generate-author-rules-map.js diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 6e40b7fd3..d61038ae5 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -253,6 +253,21 @@ jobs: cp content-temp/scripts/tina-migration/redirects.json ./ mv content-temp ./content + - name: Setup Node.js (author index) + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Setup pnpm (author index) + uses: pnpm/action-setup@v4 + with: + version: 10.10.0 + + - name: Generate author rules index + env: + LOCAL_CONTENT_RELATIVE_PATH: ../content + run: pnpm run generate:author-rules + # ────────────────────────────── AZURE LOGIN ─────────────────────────── - name: Azure CLI – Login uses: azure/login@v2 diff --git a/.gitignore b/.gitignore index 8611d4ef9..03af83784 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,4 @@ rule-to-categories.json orphaned_rules.json archived-rules.json redirects.json +public/author-title-to-rules-map.json diff --git a/README.md b/README.md index 005a79b08..d1a990794 100644 --- a/README.md +++ b/README.md @@ -99,14 +99,15 @@ To deploy a PR preview, add a comment to the PR: **`/deploy`**. This creates/upd #### In this Repository (Website) - **`build-rule-category-map.py`** - Generates two JSON files: + Generates JSON files used by the website: - `rule-to-categories.json` (maps rules to categories) - `category-uri-title-map.json` (maps category URIs to titles) - - `orphaned_rules.json` (maps category URIs to titles) + - `orphaned_rules.json` (orphaned rules report) + - `redirects.json` (old -> new URI redirects) Reads rule data from the `SSW.Rules.Content` repo and runs during the build process (via GitHub Actions) or manually from `scripts/tina-migration`. - **`prepare-content.js`** - A Node.js script that runs `build-rule-category-map.py` and moves the JSON files to the correct location for use by the website. + A Node.js script that runs the content repo Python scripts, moves the generated JSON files into this repo, and runs `pnpm run generate:author-rules` to create `public/author-title-to-rules-map.json` for `/api/tina/rules-by-author`. Uses the `LOCAL_CONTENT_RELATIVE_PATH` environment variable to locate the content repo. #### In the Content Repository diff --git a/app/api/tina/rules-by-author/route.ts b/app/api/tina/rules-by-author/route.ts index e840a1bb9..640e027d7 100644 --- a/app/api/tina/rules-by-author/route.ts +++ b/app/api/tina/rules-by-author/route.ts @@ -1,6 +1,42 @@ +import fs from "fs"; +import { join } from "path"; import { NextRequest, NextResponse } from "next/server"; import client from "@/tina/__generated__/client"; +export const runtime = "nodejs"; + +const AUTHOR_INDEX_FILENAME = "author-title-to-rules-map.json"; + +type AuthorIndex = Record; + +let cachedIndex: AuthorIndex | null | undefined; + +const getAuthorIndex = (): AuthorIndex | null => { + if (cachedIndex !== undefined) return cachedIndex; + + const indexPath = join(process.cwd(), "public", AUTHOR_INDEX_FILENAME); + try { + const raw = fs.readFileSync(indexPath, "utf8"); + cachedIndex = JSON.parse(raw) as AuthorIndex; + } catch { + cachedIndex = null; + } + + return cachedIndex; +}; + +const encodeCursor = (index: number) => Buffer.from(String(index), "utf8").toString("base64"); + +const decodeCursor = (cursor: string) => { + try { + const decoded = Buffer.from(cursor, "base64").toString("utf8"); + const n = Number(decoded); + return Number.isFinite(n) ? n : -1; + } catch { + return -1; + } +}; + export async function GET(request: NextRequest) { try { const { searchParams } = new URL(request.url); @@ -12,17 +48,76 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: "authorTitle is required" }, { status: 400 }); } - const first = firstStr ? Number(firstStr) : undefined; + const first = firstStr ? Number(firstStr) : 50; + + const index = getAuthorIndex(); + console.log("??", index) + if (!index) { + // Backward-compatible fallback (slow path) if index isn't present in the build. + const result = await client.queries.rulesByAuthor({ authorTitle, first, after }); + return NextResponse.json(result, { status: 200 }); + } + + const authorFiles = index[authorTitle] ?? []; + + const afterIndex = after ? decodeCursor(after) : -1; + if (after && afterIndex === -1) { + // Cursor from the old Tina-backed implementation. + const result = await client.queries.rulesByAuthor({ authorTitle, first, after }); + return NextResponse.json(result, { status: 200 }); + } + + const start = Math.max(0, afterIndex + 1); + const size = Number.isFinite(first) && first > 0 ? first : 50; + + const endExclusive = Math.min(start + size, authorFiles.length); + const pageFiles = authorFiles.slice(start, endExclusive); + + const nodes = await Promise.all( + pageFiles.map(async (relativePath) => { + try { + const res = await client.queries.ruleDataBasic({ relativePath }); + const rule: any = res?.data?.rule; + if (!rule || typeof rule.guid !== "string") return null; + + return { + guid: rule.guid, + title: rule.title, + uri: rule.uri, + body: rule.body, + authors: (rule.authors ?? []) + .map((a: any) => (a && a.title ? { title: a.title } : null)) + .filter((a: any) => a !== null), + lastUpdated: rule.lastUpdated, + lastUpdatedBy: rule.lastUpdatedBy, + }; + } catch { + return null; + } + }) + ); + + const edges = nodes.filter(Boolean).map((node) => ({ node })); - const result = await client.queries.rulesByAuthor({ - authorTitle, - first, - after, - }); + const hasNextPage = endExclusive < authorFiles.length; + const endCursor = edges.length > 0 ? encodeCursor(endExclusive - 1) : null; - return NextResponse.json(result, { status: 200 }); + return NextResponse.json( + { + data: { + ruleConnection: { + pageInfo: { + hasNextPage, + endCursor, + }, + edges, + }, + }, + }, + { status: 200 } + ); } catch (error) { - console.error("Error fetching rules by author from Tina:", error); + console.error("Error fetching rules by author:", error); return NextResponse.json({ error: "Failed to fetch rules", details: error instanceof Error ? error.message : String(error) }, { status: 500 }); } } diff --git a/package.json b/package.json index da087c2fb..7c70f34b2 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "lint": "biome lint", "dev:build": "next build", "prepare:content": "cross-env node ./scripts/prepare-content.js", + "generate:author-rules": "node ./scripts/generate-author-rules-map.js", "crawl-sitemap": "node ./scripts/crawl-sitemap.js" }, "devDependencies": { diff --git a/scripts/generate-author-rules-map.js b/scripts/generate-author-rules-map.js new file mode 100644 index 000000000..ddff26192 --- /dev/null +++ b/scripts/generate-author-rules-map.js @@ -0,0 +1,171 @@ +const fs = require("node:fs"); +const path = require("node:path"); + +const AUTHOR_INDEX_FILENAME = "author-title-to-rules-map.json"; + +const parseArgs = (argv) => { + const args = {}; + for (let i = 2; i < argv.length; i++) { + const current = argv[i]; + if (!current?.startsWith("--")) continue; + + const key = current.slice(2); + const next = argv[i + 1]; + const value = next && !next.startsWith("--") ? next : "true"; + + args[key] = value; + if (value !== "true") i++; + } + return args; +}; + +const unquote = (value) => { + const trimmed = String(value).trim(); + if ((trimmed.startsWith("\"") && trimmed.endsWith("\"")) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) { + return trimmed.slice(1, -1).trim(); + } + return trimmed; +}; + +const extractAuthorTitles = (rawMdx) => { + const match = rawMdx.match(/^---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|$)/); + if (!match) return []; + + const frontmatter = match[1]; + const lines = frontmatter.split(/\r?\n/); + + let inAuthors = false; + const titles = []; + + for (const line of lines) { + if (!inAuthors) { + if (line.trimEnd() === "authors:") { + inAuthors = true; + } + continue; + } + + // Stop when we hit a new top-level key + if (/^[A-Za-z0-9_]+\s*:/.test(line) && !/^\s*-/.test(line)) { + break; + } + + const titleKvp = line.match(/^\s*-\s*title:\s*(.+?)\s*$/); + if (titleKvp) { + titles.push(unquote(titleKvp[1])); + continue; + } + + const titleInlineObj = line.match(/^\s*-\s*\{\s*title:\s*([^,}]+)[,}]/); + if (titleInlineObj) { + titles.push(unquote(titleInlineObj[1])); + continue; + } + + // Support string list items: - Bob Northwind + const stringItem = line.match(/^\s*-\s*(.+?)\s*$/); + if (stringItem && !stringItem[1].includes(":")) { + titles.push(unquote(stringItem[1])); + } + } + + return titles.filter((t) => t.length > 0); +}; + +const listRuleFiles = (rulesDirAbs) => { + const results = []; + const stack = [{ abs: rulesDirAbs, relPosix: "" }]; + + while (stack.length > 0) { + const { abs, relPosix } = stack.pop(); + const entries = fs.readdirSync(abs, { withFileTypes: true }); + + for (const entry of entries) { + if (entry.isDirectory()) { + const nextAbs = path.join(abs, entry.name); + const nextRelPosix = relPosix ? `${relPosix}/${entry.name}` : entry.name; + stack.push({ abs: nextAbs, relPosix: nextRelPosix }); + } else if (entry.isFile() && entry.name === "rule.mdx") { + results.push(relPosix ? `${relPosix}/rule.mdx` : "rule.mdx"); + } + } + } + + return results.sort(); +}; + +const looksLikeContentRoot = (dir) => { + if (!dir) return false; + try { + return fs.existsSync(path.join(dir, "public", "uploads", "rules")); + } catch { + return false; + } +}; + +const resolveContentRoot = (args, repoRoot) => { + const candidates = []; + + if (args.contentRoot) { + candidates.push(path.resolve(args.contentRoot)); + } + + const rel = process.env.LOCAL_CONTENT_RELATIVE_PATH; + if (rel) { + // Support paths relative to either repo root or this scripts/ directory. + candidates.push(path.resolve(__dirname, rel)); + candidates.push(path.resolve(repoRoot, rel)); + } + + // Common local/CI layouts + candidates.push(path.join(repoRoot, "content")); + candidates.push(path.join(repoRoot, "..", "SSW.Rules.Content")); + + const deduped = Array.from(new Set(candidates)); + for (const candidate of deduped) { + if (looksLikeContentRoot(candidate)) return candidate; + } + + console.error("Could not locate the content repo (missing LOCAL_CONTENT_RELATIVE_PATH and auto-detection failed).\n"); + console.error("Fix options:"); + console.error(" 1) Set LOCAL_CONTENT_RELATIVE_PATH (e.g. ../SSW.Rules.Content or ./content)"); + console.error(" 2) Or pass --contentRoot (e.g. pnpm run generate:author-rules -- --contentRoot ..\\SSW.Rules.Content)\n"); + console.error("Tried:"); + for (const candidate of deduped) { + console.error(` - ${candidate}`); + } + process.exit(1); +}; + +const main = () => { + const args = parseArgs(process.argv); + + const repoRoot = path.resolve(__dirname, ".."); + const outFile = args.outFile ? path.resolve(args.outFile) : path.join(repoRoot, "public", AUTHOR_INDEX_FILENAME); + + const contentRoot = resolveContentRoot(args, repoRoot); + + const rulesDir = path.resolve(contentRoot, "public", "uploads", "rules"); + const ruleFiles = listRuleFiles(rulesDir); + const index = {}; + + for (const relativePathPosix of ruleFiles) { + const absFile = path.join(rulesDir, ...relativePathPosix.split("/")); + const raw = fs.readFileSync(absFile, "utf8"); + const titles = extractAuthorTitles(raw); + + for (const title of titles) { + (index[title] ||= []).push(relativePathPosix); + } + } + + for (const [title, files] of Object.entries(index)) { + index[title] = Array.from(new Set(files)).sort(); + } + + fs.mkdirSync(path.dirname(outFile), { recursive: true }); + fs.writeFileSync(outFile, `${JSON.stringify(index, null, 2)}\n`, "utf8"); + console.log(`Generated ${outFile} (${Object.keys(index).length} authors)`); +}; + +main(); diff --git a/scripts/prepare-content-build.js b/scripts/prepare-content-build.js index 9218dc4f5..ece47c339 100644 --- a/scripts/prepare-content-build.js +++ b/scripts/prepare-content-build.js @@ -88,4 +88,8 @@ copyAndMoveJsonFile("category-uri-title-map.json", scriptsPath, destDir); copyAndMoveJsonFile("orphaned_rules.json", scriptsPath, destDir); copyAndMoveJsonFile("redirects.json", scriptsPath, destDir); +const generateAuthorRulesMapScript = join(destDir, "scripts", "generate-author-rules-map.js"); +const outFile = join(destDir, "public", "author-title-to-rules-map.json"); +execSync(`node "${generateAuthorRulesMapScript}" --contentRoot "${tempDir}" --outFile "${outFile}"`, { stdio: "inherit" }); + console.log("Content preparation completed successfully!"); diff --git a/scripts/prepare-content.js b/scripts/prepare-content.js index 372455af4..44fc847a3 100644 --- a/scripts/prepare-content.js +++ b/scripts/prepare-content.js @@ -60,3 +60,7 @@ execSync(`${python} "${buildRedirectMapScript}"`, { stdio: "inherit", cwd: scrip copyAndMoveJsonFile("category-uri-title-map.json", scriptsPath); copyAndMoveJsonFile("orphaned_rules.json", scriptsPath); copyAndMoveJsonFile("redirects.json", scriptsPath); + +const generateAuthorRulesMapScript = resolve(__dirname, "generate-author-rules-map.js"); +const outFile = resolve(__dirname, "../public/author-title-to-rules-map.json"); +execSync(`node "${generateAuthorRulesMapScript}" --contentRoot "${contentAbsPath}" --outFile "${outFile}"`, { stdio: "inherit" });