diff --git a/README.md b/README.md index 14f36f9..0c91b8e 100644 --- a/README.md +++ b/README.md @@ -29,18 +29,20 @@ It runs a **4-stage AI pipeline** on your bookmarks: ↓ 🏷️ Entity Extraction β€” mines hashtags, URLs, mentions, and 100+ known tools from raw tweet data (free, zero API calls) ↓ -πŸ‘οΈ Vision Analysis β€” reads text, objects, and context from every image/GIF/video thumbnail (30–40 visual tags per image) +πŸ‘οΈ Vision Analysis β€” reads text, objects, and context from every image/GIF/video thumbnail ↓ -🧠 Semantic Tagging β€” generates 25–35 searchable tags per bookmark for AI-powered search +🧠 Semantic Tagging β€” generates broad, reusable search tags per bookmark for AI-powered search ↓ πŸ“‚ Categorization β€” assigns each bookmark to 1–3 categories with confidence scores + ↓ +πŸ“– Obsidian Export β€” writes your knowledge base as Markdown notes with YAML frontmatter, wikilinks, and index files ``` After the pipeline runs, you get: - **AI search** β€” find bookmarks by meaning, not just keywords (*"funny meme about crypto crashing"*) - **Interactive mindmap** β€” explore your entire bookmark graph visually - **Filtered browsing** β€” grid or list view, filter by category, media type, and date -- **Export tools** β€” download media, export as CSV / JSON / ZIP +- **Export tools** β€” download media, export as CSV / JSON / ZIP / Obsidian vault --- @@ -224,6 +226,45 @@ Create custom categories with a name, color, and optional description. The descr - **CSV** β€” spreadsheet-compatible with all fields - **JSON** β€” full structured data export - **ZIP** β€” exports a category's bookmarks + all media files with a `manifest.csv` +- **Obsidian** β€” exports your entire knowledge base as a Markdown vault (see below) + +### πŸ“– Obsidian Export + +Export all processed bookmarks directly into an [Obsidian](https://obsidian.md) vault as structured Markdown notes. + +**Each bookmark becomes a note** with YAML frontmatter: + +```yaml +--- +tweet_id: "1933508347334177246" +author: "alex_prompter" +author_name: "Alex Prompter" +date: 2025-06-13 +source: "https://x.com/alex_prompter/status/1933508347334177246" +categories: ["AI & Agents", "PKM & Workflows"] +tags: + - twitter/bookmark + - author/alex_prompter + - prompt-engineering + - llm + - automation + - category/ai-agents +--- +``` + +**Index notes** are generated automatically in a `_index/` subfolder: +- One note per **category** β€” lists all bookmarks in that category as `[[wikilinks]]` +- One note per **author** β€” lists every bookmark from that person + +This creates a dense backlink graph in Obsidian's graph view, where bookmarks cluster naturally by topic and author. + +**To set up:** +1. Go to **Settings** β†’ find the **Obsidian Export** section +2. Enter the full path to your Obsidian vault folder (e.g. `/Users/you/Obsidian/Personal`) +3. Click **Save**, then **Export to Obsidian** +4. Open your vault in Obsidian β€” notes appear in a `Twitter Bookmarks/` folder + +Re-export anytime. By default, existing notes are skipped β€” enable **Overwrite** to replace them. ### ⌨️ Command Palette @@ -241,6 +282,7 @@ All settings are manageable in the **Settings** page at `/settings` or via envir | API Base URL | `ANTHROPIC_BASE_URL` | Custom endpoint for proxies or local Anthropic-compatible models | | AI Model | Settings page only | Haiku 4.5 (default, fastest/cheapest), Sonnet 4.6, Opus 4.6 | | OpenAI Key | Settings page only | Alternative provider if no Anthropic key is set | +| Obsidian Vault Path | Settings page only | Absolute path to your Obsidian vault folder for Markdown export | | Database | `DATABASE_URL` | SQLite file path (default: `file:./prisma/dev.db`) | ### Custom API Endpoint @@ -266,6 +308,7 @@ siftly/ β”‚ β”‚ β”‚ └── [slug]/ # Individual category operations β”‚ β”‚ β”œβ”€β”€ categorize/ # 4-stage AI pipeline (start, status, stop) β”‚ β”‚ β”œβ”€β”€ export/ # CSV, JSON, ZIP export +β”‚ β”‚ β”‚ └── obsidian/ # Obsidian vault export endpoint β”‚ β”‚ β”œβ”€β”€ import/ # JSON file import with dedup + auto-pipeline trigger β”‚ β”‚ β”‚ β”œβ”€β”€ bookmarklet/ # Bookmarklet-specific import endpoint β”‚ β”‚ β”‚ └── twitter/ # Twitter-specific import endpoint @@ -308,6 +351,7 @@ siftly/ β”‚ β”œβ”€β”€ rawjson-extractor.ts # Entity extraction from raw tweet JSON β”‚ β”œβ”€β”€ parser.ts # Multi-format JSON parser β”‚ β”œβ”€β”€ exporter.ts # CSV, JSON, ZIP export +β”‚ β”œβ”€β”€ obsidian-exporter.ts # Obsidian vault export (Markdown + YAML frontmatter + indexes) β”‚ β”œβ”€β”€ types.ts # Shared TypeScript types β”‚ └── db.ts # Prisma client singleton β”‚ diff --git a/app/api/export/obsidian/route.ts b/app/api/export/obsidian/route.ts new file mode 100644 index 0000000..d4327a8 --- /dev/null +++ b/app/api/export/obsidian/route.ts @@ -0,0 +1,38 @@ +import { NextRequest, NextResponse } from 'next/server' +import { exportToObsidian } from '@/lib/obsidian-exporter' +import prisma from '@/lib/db' + +export async function POST(request: NextRequest): Promise { + let body: { category?: string; overwrite?: boolean } = {} + try { + body = await request.json() + } catch { + // body stays as defaults + } + + const { category, overwrite = false } = body + + const setting = await prisma.setting.findUnique({ where: { key: 'obsidianVaultPath' } }) + if (!setting?.value) { + return NextResponse.json( + { error: 'Obsidian vault path not configured. Add it in Settings.' }, + { status: 400 } + ) + } + + try { + const result = await exportToObsidian({ + vaultPath: setting.value, + subfolder: 'Twitter Bookmarks', + overwrite, + categoryFilter: category, + }) + return NextResponse.json(result) + } catch (err: unknown) { + console.error('Obsidian export error:', err) + return NextResponse.json( + { error: `Export failed: ${err instanceof Error ? err.message : String(err)}` }, + { status: 500 } + ) + } +} diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index f06373e..88cb84e 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -24,7 +24,7 @@ const ALLOWED_OPENAI_MODELS = [ export async function GET(): Promise { try { - const [anthropic, anthropicModel, provider, openai, openaiModel, xClientId, xClientSecret] = await Promise.all([ + const [anthropic, anthropicModel, provider, openai, openaiModel, xClientId, xClientSecret, obsidianVaultPath] = await Promise.all([ prisma.setting.findUnique({ where: { key: 'anthropicApiKey' } }), prisma.setting.findUnique({ where: { key: 'anthropicModel' } }), prisma.setting.findUnique({ where: { key: 'aiProvider' } }), @@ -32,6 +32,7 @@ export async function GET(): Promise { prisma.setting.findUnique({ where: { key: 'openaiModel' } }), prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }), prisma.setting.findUnique({ where: { key: 'x_oauth_client_secret' } }), + prisma.setting.findUnique({ where: { key: 'obsidianVaultPath' } }), ]) return NextResponse.json({ @@ -45,6 +46,7 @@ export async function GET(): Promise { xOAuthClientId: maskKey(xClientId?.value ?? null), xOAuthClientSecret: maskKey(xClientSecret?.value ?? null), hasXOAuth: !!xClientId?.value, + obsidianVaultPath: obsidianVaultPath?.value ?? null, }) } catch (err) { console.error('Settings GET error:', err) @@ -161,6 +163,20 @@ export async function POST(request: NextRequest): Promise { } } + // Save Obsidian vault path if provided + if ('obsidianVaultPath' in body) { + const vaultPath = (body as { obsidianVaultPath?: string }).obsidianVaultPath + if (typeof vaultPath !== 'string' || vaultPath.trim() === '') { + return NextResponse.json({ error: 'Invalid obsidianVaultPath value' }, { status: 400 }) + } + await prisma.setting.upsert({ + where: { key: 'obsidianVaultPath' }, + update: { value: vaultPath.trim() }, + create: { key: 'obsidianVaultPath', value: vaultPath.trim() }, + }) + return NextResponse.json({ saved: true }) + } + // Save X OAuth credentials if provided const { xOAuthClientId, xOAuthClientSecret } = body const xKeys: { key: string; value: string | undefined }[] = [ diff --git a/app/api/settings/test/route.ts b/app/api/settings/test/route.ts index 84c12af..d318328 100644 --- a/app/api/settings/test/route.ts +++ b/app/api/settings/test/route.ts @@ -53,7 +53,7 @@ export async function POST(request: NextRequest): Promise { let client try { - client = resolveOpenAIClient({ dbKey }) + client = await resolveOpenAIClient({ dbKey }) } catch { return NextResponse.json({ working: false, error: 'No OpenAI API key found. Add one in Settings or set up Codex CLI.' }) } diff --git a/app/import/page.tsx b/app/import/page.tsx index 255bf46..8588b55 100644 --- a/app/import/page.tsx +++ b/app/import/page.tsx @@ -92,7 +92,16 @@ const BOOKMARKLET_SCRIPT = `(async function(){ function addTweet(t){ if(!t||!t.rest_id||seen.has(t.rest_id))return; seen.add(t.rest_id); - var leg=t.legacy||{},usr=(t.core&&t.core.user_results&&t.core.user_results.result&&t.core.user_results.result.legacy)||{}; + var leg=t.legacy||{}; + var res=t.core&&t.core.user_results&&t.core.user_results.result; + var usrLeg=(res&&res.legacy)||{}; + var usrCore=(res&&res.core)||{}; + var usrAvatar=(res&&res.avatar)||{}; + var usr={ + name:usrCore.name||usrLeg.name||'Unknown', + screen_name:usrCore.screen_name||usrLeg.screen_name||'unknown', + profile_image_url_https:usrAvatar.image_url||usrLeg.profile_image_url_https||'' + }; var rawMedia=(leg.extended_entities&&leg.extended_entities.media)||(leg.entities&&leg.entities.media)||[]; var media=rawMedia.map(function(m){ var thumb=m.media_url_https||''; @@ -191,11 +200,12 @@ const BOOKMARKLET_SCRIPT = `(async function(){ document.body.appendChild(btn); document.body.appendChild(autoBtn); var origFetch=window.fetch; + function isRelevantUrl(u){return u.includes('/graphql/')&&(isLikes?u.toLowerCase().includes('like'):u.includes('Bookmark'));} window.fetch=async function(){ var r=await origFetch.apply(this,arguments); try{ var u=arguments[0] instanceof Request?arguments[0].url:String(arguments[0]); - if(u.includes('/graphql/')){var d=await r.clone().json();processData(d);} + if(isRelevantUrl(u)){var d=await r.clone().json();processData(d);} }catch(ex){} return r; }; @@ -203,7 +213,7 @@ const BOOKMARKLET_SCRIPT = `(async function(){ XMLHttpRequest.prototype.open=function(){xhrUrls.set(this,String(arguments[1]||''));return origOpen.apply(this,arguments);}; XMLHttpRequest.prototype.send=function(){ var xhr=this,u=xhrUrls.get(xhr)||''; - if(u.includes('/graphql/')){xhr.addEventListener('load',function(){try{processData(JSON.parse(xhr.responseText));}catch(ex){}});} + if(isRelevantUrl(u)){xhr.addEventListener('load',function(){try{processData(JSON.parse(xhr.responseText));}catch(ex){}});} return origSend.apply(this,arguments); }; showToast('\u2705 Active! Scroll your '+label+' \u2014 counter updates above.','#1e1b4b'); @@ -222,7 +232,16 @@ const CONSOLE_SCRIPT = `(async function() { function addTweet(t) { if (!t?.rest_id || seen.has(t.rest_id)) return; seen.add(t.rest_id); - const leg = t.legacy ?? {}, usr = t.core?.user_results?.result?.legacy ?? {}; + const leg = t.legacy ?? {}; + const res = t.core?.user_results?.result; + const usrLeg = res?.legacy ?? {}; + const usrCore = res?.core ?? {}; + const usrAvatar = res?.avatar ?? {}; + const usr = { + name: usrCore.name ?? usrLeg.name ?? 'Unknown', + screen_name: usrCore.screen_name ?? usrLeg.screen_name ?? 'unknown', + profile_image_url_https: usrAvatar.image_url ?? usrLeg.profile_image_url_https ?? '', + }; const media = (leg.extended_entities?.media ?? leg.entities?.media ?? []).map(m => { const thumb = m.media_url_https ?? ''; if (m.type === 'video' || m.type === 'animated_gif') { @@ -336,14 +355,13 @@ const CONSOLE_SCRIPT = `(async function() { autoBtn.style.background = '#4f46e5'; autoBtn.style.color = '#fff'; autoBtn.style.border = 'none'; runAutoScroll(); }; - document.body.appendChild(btn); - document.body.appendChild(autoBtn); + const isRelevantUrl = (u) => u.includes('/graphql/') && (isLikes ? u.toLowerCase().includes('like') : u.includes('Bookmark')); const origFetch = window.fetch; window.fetch = async function(...args) { const r = await origFetch.apply(this, args); try { const u = args[0] instanceof Request ? args[0].url : String(args[0]); - if (u.includes('/graphql/')) { + if (isRelevantUrl(u)) { const d = await r.clone().json(); processData(d); } @@ -359,13 +377,15 @@ const CONSOLE_SCRIPT = `(async function() { }; XMLHttpRequest.prototype.send = function(...args) { const xhr = this, u = xhrUrls.get(xhr) ?? ''; - if (u.includes('/graphql/')) { + if (isRelevantUrl(u)) { xhr.addEventListener('load', function() { try { processData(JSON.parse(xhr.responseText)); } catch(e) {} }); } return origSend.apply(this, args); }; + document.body.appendChild(btn); + document.body.appendChild(autoBtn); console.log(\`βœ… Script active. Scroll through your \${label}, then click the purple button.\`); })();` diff --git a/app/settings/page.tsx b/app/settings/page.tsx index f022624..61d1ef6 100644 --- a/app/settings/page.tsx +++ b/app/settings/page.tsx @@ -20,6 +20,8 @@ import { Terminal, Loader2, X, + BookOpen, + FolderOpen, } from 'lucide-react' const ANTHROPIC_MODELS = [ @@ -655,7 +657,164 @@ function ExportButton({ ) } -function DataSection() { +interface ObsidianResult { + written: number + skipped: number + errors: Array<{ tweetId: string; error: string }> + indexesWritten: number +} + +function ObsidianExportBlock({ onToast }: { onToast: (t: Toast) => void }) { + const [vaultPath, setVaultPath] = useState('') + const [savedPath, setSavedPath] = useState(null) + const [savingPath, setSavingPath] = useState(false) + const [exporting, setExporting] = useState(false) + const [result, setResult] = useState(null) + const [overwrite, setOverwrite] = useState(false) + + useEffect(() => { + fetch('/api/settings') + .then((r) => r.json()) + .then((d: Record) => { + if (d.obsidianVaultPath) setSavedPath(d.obsidianVaultPath as string) + }) + .catch(() => {}) + }, []) + + async function handleSavePath() { + if (!vaultPath.trim()) { + onToast({ type: 'error', message: 'Enter a vault path first' }) + return + } + setSavingPath(true) + try { + const res = await fetch('/api/settings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ obsidianVaultPath: vaultPath.trim() }), + }) + if (!res.ok) { + const data = await res.json() as { error?: string } + throw new Error(data.error ?? 'Failed to save') + } + setSavedPath(vaultPath.trim()) + setVaultPath('') + onToast({ type: 'success', message: 'Vault path saved' }) + } catch (err) { + onToast({ type: 'error', message: err instanceof Error ? err.message : 'Failed to save path' }) + } finally { + setSavingPath(false) + } + } + + async function handleExport() { + setExporting(true) + setResult(null) + try { + const res = await fetch('/api/export/obsidian', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ overwrite }), + }) + const data = await res.json() as ObsidianResult & { error?: string } + if (!res.ok) throw new Error(data.error ?? 'Export failed') + setResult(data) + onToast({ type: 'success', message: `${data.written} notes written to vault` }) + } catch (err) { + onToast({ type: 'error', message: err instanceof Error ? err.message : 'Export failed' }) + } finally { + setExporting(false) + } + } + + return ( +
+
+ +

Export to Obsidian

+
+ + {/* Vault path row */} +
+
+
+ + setVaultPath(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && void handleSavePath()} + placeholder={savedPath ?? 'C:\\Users\\you\\Obsidian\\MyVault'} + className="w-full pl-8 pr-3.5 py-2.5 rounded-xl bg-zinc-800 border border-zinc-700 text-zinc-100 placeholder:text-zinc-600 text-sm focus:outline-none focus:border-indigo-500 focus:ring-1 focus:ring-indigo-500/20 transition-all duration-200 font-mono" + /> +
+ +
+ {savedPath && ( +

+ + {savedPath} +

+ )} +
+ + {/* Overwrite toggle + export button */} +
+ + +
+ + {/* Result summary */} + {result && ( +
+ + {result.written} written + + + {result.skipped} skipped + + + {result.indexesWritten} indexes + + {result.errors.length > 0 && ( + + {result.errors.length} errors + + )} +
+ )} + +

+ Notes are written to a Twitter Bookmarks/ subfolder inside your vault. Index notes linking by author and category are created automatically in _index/. +

+
+ ) +} + +function DataSection({ onToast }: { onToast: (t: Toast) => void }) { return (
+
) } @@ -1006,7 +1166,7 @@ export default function SettingsPage() {
- +
diff --git a/lib/ai-client.ts b/lib/ai-client.ts index ca8135b..5c94110 100644 --- a/lib/ai-client.ts +++ b/lib/ai-client.ts @@ -1,5 +1,5 @@ import Anthropic from '@anthropic-ai/sdk' -import OpenAI from 'openai' +import type OpenAI from 'openai' import { resolveAnthropicClient } from './claude-cli-auth' import { resolveOpenAIClient } from './openai-auth' import { getProvider } from './settings' @@ -106,7 +106,7 @@ export async function resolveAIClient(options: { const provider = await getProvider() if (provider === 'openai') { - const client = resolveOpenAIClient(options) + const client = await resolveOpenAIClient(options) return new OpenAIAIClient(client) } diff --git a/lib/obsidian-exporter.ts b/lib/obsidian-exporter.ts new file mode 100644 index 0000000..1185fa3 --- /dev/null +++ b/lib/obsidian-exporter.ts @@ -0,0 +1,252 @@ +import fs from 'fs/promises' +import path from 'path' +import prisma from '@/lib/db' + +export interface ObsidianExportResult { + written: number + skipped: number + errors: Array<{ tweetId: string; error: string }> + indexesWritten: number +} + +interface ObsidianExportOptions { + vaultPath: string + subfolder?: string + overwrite?: boolean + categoryFilter?: string +} + +interface MediaItemRow { + type: string + url: string + thumbnailUrl: string | null +} + +interface CategoryJoin { + category: { + name: string + slug: string + color: string + } +} + +interface BookmarkRow { + id: string + tweetId: string + text: string + authorHandle: string + authorName: string + tweetCreatedAt: Date | null + importedAt: Date + semanticTags: string | null + entities: string | null + mediaItems: MediaItemRow[] + categories: CategoryJoin[] +} + +function sanitizeTag(tag: string): string { + return tag + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9\-_/]/g, '') + .replace(/-+/g, '-') + .trim() +} + +function sanitizeFilename(str: string): string { + return str.replace(/[<>:"/\\|?*\n\r]/g, '').trim() +} + +function noteFilename(bookmark: BookmarkRow): string { + const date = bookmark.tweetCreatedAt + ? new Date(bookmark.tweetCreatedAt).toISOString().split('T')[0] + : 'unknown' + const author = sanitizeFilename(bookmark.authorHandle || 'unknown') + return `${date} - @${author} - ${bookmark.tweetId}.md` +} + +function buildNoteMarkdown(bookmark: BookmarkRow): string { + const tags: string[] = ['twitter/bookmark'] + + if (bookmark.authorHandle) { + tags.push(`author/${sanitizeTag(bookmark.authorHandle)}`) + } + + let semanticTags: string[] = [] + try { semanticTags = JSON.parse(bookmark.semanticTags || '[]') } catch {} + semanticTags.forEach(t => { const c = sanitizeTag(t); if (c) tags.push(c) }) + + const categories = bookmark.categories.map((bc) => bc.category.name) + categories.forEach((c) => tags.push(`category/${sanitizeTag(c)}`)) + + const date = bookmark.tweetCreatedAt + ? new Date(bookmark.tweetCreatedAt).toISOString().split('T')[0] + : null + const sourceUrl = `https://x.com/${bookmark.authorHandle}/status/${bookmark.tweetId}` + + const frontmatter = [ + '---', + `tweet_id: "${bookmark.tweetId}"`, + `author: "${bookmark.authorHandle || ''}"`, + `author_name: "${(bookmark.authorName || '').replace(/"/g, "'")}"`, + date ? `date: ${date}` : null, + `source: "${sourceUrl}"`, + `categories: [${categories.map((c) => `"${c}"`).join(', ')}]`, + `tags:`, + ...tags.map(t => ` - ${t}`), + '---', + ].filter(Boolean).join('\n') + + const lines: string[] = [frontmatter, '', bookmark.text || ''] + + if (bookmark.mediaItems.length > 0) { + lines.push('', '## Media', '') + for (const item of bookmark.mediaItems) { + if (item.type === 'photo') { + lines.push(`![](${item.url})`) + } else { + lines.push(`[${item.type.toUpperCase()}](${item.url})`) + } + } + } + + lines.push('', '## Source', '', `[View on X](${sourceUrl})`) + return lines.join('\n') +} + +// Build a category index note with wikilinks to every bookmark in that category +function buildCategoryIndex( + categoryName: string, + bookmarks: BookmarkRow[] +): string { + const tag = sanitizeTag(categoryName) + const links = bookmarks + .map(b => `- [[${noteFilename(b).replace(/\.md$/, '')}]]`) + .join('\n') + + return [ + '---', + `type: index`, + `category: "${categoryName}"`, + `tags:`, + ` - index/category`, + ` - category/${tag}`, + '---', + '', + `# ${categoryName}`, + '', + `${bookmarks.length} bookmarks`, + '', + '## Bookmarks', + '', + links, + ].join('\n') +} + +// Build an author index note with wikilinks to every bookmark from that author +function buildAuthorIndex( + handle: string, + displayName: string, + bookmarks: BookmarkRow[] +): string { + const links = bookmarks + .map(b => `- [[${noteFilename(b).replace(/\.md$/, '')}]]`) + .join('\n') + + return [ + '---', + `type: index`, + `author: "${handle}"`, + `author_name: "${displayName.replace(/"/g, "'")}"`, + `tags:`, + ` - index/author`, + ` - author/${sanitizeTag(handle)}`, + '---', + '', + `# @${handle}`, + '', + `${bookmarks.length} bookmarks`, + '', + '## Bookmarks', + '', + links, + ].join('\n') +} + +export async function exportToObsidian(options: ObsidianExportOptions): Promise { + const { vaultPath, subfolder = 'Twitter Bookmarks', overwrite = false, categoryFilter } = options + + const notesDir = path.join(vaultPath, subfolder) + const indexDir = path.join(vaultPath, subfolder, '_index') + await fs.mkdir(notesDir, { recursive: true }) + await fs.mkdir(indexDir, { recursive: true }) + + const where = categoryFilter + ? { categories: { some: { category: { slug: categoryFilter } } } } + : {} + + const bookmarks = await prisma.bookmark.findMany({ + where, + include: { + mediaItems: true, + categories: { include: { category: true } }, + }, + orderBy: { tweetCreatedAt: 'desc' }, + }) as BookmarkRow[] + + const result: ObsidianExportResult = { written: 0, skipped: 0, errors: [], indexesWritten: 0 } + + // Write individual bookmark notes + for (const bookmark of bookmarks) { + const filename = noteFilename(bookmark) + const filePath = path.join(notesDir, filename) + + if (!overwrite) { + try { await fs.access(filePath); result.skipped++; continue } catch {} + } + + try { + await fs.writeFile(filePath, buildNoteMarkdown(bookmark), 'utf-8') + result.written++ + } catch (err: unknown) { + result.errors.push({ + tweetId: bookmark.tweetId, + error: err instanceof Error ? err.message : String(err), + }) + } + } + + // Build and write category index notes + const byCategory = new Map() + for (const bookmark of bookmarks) { + for (const bc of bookmark.categories) { + const name = bc.category.name + if (!byCategory.has(name)) byCategory.set(name, []) + byCategory.get(name)!.push(bookmark) + } + } + for (const [categoryName, categoryBookmarks] of byCategory) { + const filename = `_${sanitizeFilename(categoryName)}.md` + const filePath = path.join(indexDir, filename) + await fs.writeFile(filePath, buildCategoryIndex(categoryName, categoryBookmarks), 'utf-8') + result.indexesWritten++ + } + + // Build and write author index notes + const byAuthor = new Map() + for (const bookmark of bookmarks) { + const handle = bookmark.authorHandle || 'unknown' + if (!byAuthor.has(handle)) { + byAuthor.set(handle, { displayName: bookmark.authorName || handle, bookmarks: [] }) + } + byAuthor.get(handle)!.bookmarks.push(bookmark) + } + for (const [handle, { displayName, bookmarks: authorBookmarks }] of byAuthor) { + const filename = `@${sanitizeFilename(handle)}.md` + const filePath = path.join(indexDir, filename) + await fs.writeFile(filePath, buildAuthorIndex(handle, displayName, authorBookmarks), 'utf-8') + result.indexesWritten++ + } + + return result +} diff --git a/lib/openai-auth.ts b/lib/openai-auth.ts index ff77ca9..34a0848 100644 --- a/lib/openai-auth.ts +++ b/lib/openai-auth.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs' import { join } from 'path' import { homedir } from 'os' -import OpenAI from 'openai' +import type OpenAI from 'openai' interface CodexAuth { auth_mode?: string @@ -104,17 +104,19 @@ export function getCodexCliAuthStatus(): { return { available: true, authMode: auth.auth_mode, planType } } -function createCodexOpenAIClient(baseURL?: string): OpenAI | null { +async function createCodexOpenAIClient(baseURL?: string): Promise { const apiKey = getCodexApiKey() if (!apiKey) return null + const { default: OpenAI } = await import('openai') return new OpenAI({ apiKey, ...(baseURL ? { baseURL } : {}) }) } -export function resolveOpenAIClient(options: { +export async function resolveOpenAIClient(options: { overrideKey?: string dbKey?: string baseURL?: string -} = {}): OpenAI { +} = {}): Promise { + const { default: OpenAI } = await import('openai') const baseURL = options.baseURL ?? process.env.OPENAI_BASE_URL if (options.overrideKey?.trim()) { @@ -125,7 +127,7 @@ export function resolveOpenAIClient(options: { return new OpenAI({ apiKey: options.dbKey.trim(), ...(baseURL ? { baseURL } : {}) }) } - const cliClient = createCodexOpenAIClient(baseURL) + const cliClient = await createCodexOpenAIClient(baseURL) if (cliClient) return cliClient const envKey = process.env.OPENAI_API_KEY?.trim()