feat(import): implement X OAuth 2.0 backend for live bookmark import

.env.example

@@ -22,6 +22,13 @@ DATABASE_URL="file:./prisma/dev.db"
 # Optional: custom API base URL (proxy or local model server)
 # ANTHROPIC_BASE_URL=
 
+# ── X OAuth 2.0 (optional) ──────────────────────────────────────────
+
+# Base URL of your Siftly instance. Used to construct the OAuth callback URL.
+# Required if running behind a reverse proxy or at a non-default port.
+# Defaults to http://localhost:3000 if unset.
+# APP_URL=http://localhost:3000
+
 # ── Access control (optional) ────────────────────────────────────────
 
 # Set BOTH to enable HTTP Basic Auth on the entire app.

app/api/import/x-oauth/authorize/route.ts

@@ -0,0 +1,48 @@
+import { NextResponse } from 'next/server'
+import prisma from '@/lib/db'
+import crypto from 'crypto'
+
+function base64url(buf: Buffer): string {
+  return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '')
+}
+
+const APP_URL = process.env.APP_URL ?? 'http://localhost:3000'
+const CALLBACK_URL = `${APP_URL}/api/import/x-oauth/callback`
+const SCOPES = 'bookmark.read tweet.read users.read offline.access'
+
+export async function GET() {
+  try {
+    const clientId = await prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } })
+    if (!clientId?.value) {
+      return NextResponse.json({ error: 'X OAuth Client ID not configured in Settings' }, { status: 400 })
+    }
+
+    const codeVerifier = base64url(crypto.randomBytes(32))
+    const codeChallenge = base64url(crypto.createHash('sha256').update(codeVerifier).digest())
+    const state = base64url(crypto.randomBytes(16))
+
+    // Store PKCE verifier keyed by state (cleaned up after callback)
+    await prisma.setting.upsert({
+      where: { key: `x_oauth_pkce_${state}` },
+      update: { value: codeVerifier },
+      create: { key: `x_oauth_pkce_${state}`, value: codeVerifier },
+    })
+
+    const params = new URLSearchParams({
+      response_type: 'code',
+      client_id: clientId.value,
+      redirect_uri: CALLBACK_URL,
+      scope: SCOPES,
+      state,
+      code_challenge: codeChallenge,
+      code_challenge_method: 'S256',
+    })
+
+    return NextResponse.json({ authUrl: `https://twitter.com/i/oauth2/authorize?${params}` })
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : 'Failed to build auth URL' },
+      { status: 500 },
+    )
+  }
+}

app/api/import/x-oauth/callback/route.ts

@@ -0,0 +1,93 @@
+import { NextRequest, NextResponse } from 'next/server'
+import prisma from '@/lib/db'
+
+const APP_URL = process.env.APP_URL ?? 'http://localhost:3000'
+const CALLBACK_URL = `${APP_URL}/api/import/x-oauth/callback`
+const IMPORT_URL = `${APP_URL}/import`
+
+export async function GET(request: NextRequest): Promise<NextResponse> {
+  const { searchParams } = new URL(request.url)
+  const code = searchParams.get('code')
+  const state = searchParams.get('state')
+  const error = searchParams.get('error')
+
+  if (error) {
+    return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(error)}`)
+  }
+
+  if (!code || !state) {
+    return NextResponse.redirect(`${IMPORT_URL}?x_error=missing_params`)
+  }
+
+  try {
+    // Retrieve and clean up PKCE verifier
+    const verifierKey = `x_oauth_pkce_${state}`
+    const verifierSetting = await prisma.setting.findUnique({ where: { key: verifierKey } })
+    if (!verifierSetting?.value) {
+      return NextResponse.redirect(`${IMPORT_URL}?x_error=invalid_state`)
+    }
+    const codeVerifier = verifierSetting.value
+    await prisma.setting.delete({ where: { key: verifierKey } }).catch(() => {})
+
+    const [clientIdSetting, clientSecretSetting] = await Promise.all([
+      prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_client_secret' } }),
+    ])
+    if (!clientIdSetting?.value) {
+      return NextResponse.redirect(`${IMPORT_URL}?x_error=client_id_missing`)
+    }
+
+    // Exchange auth code for tokens
+    const tokenBody = new URLSearchParams({
+      code,
+      grant_type: 'authorization_code',
+      client_id: clientIdSetting.value,
+      redirect_uri: CALLBACK_URL,
+      code_verifier: codeVerifier,
+    })
+
+    // Confidential clients send Basic auth; public clients omit it
+    const authHeader = clientSecretSetting?.value
+      ? 'Basic ' + Buffer.from(`${clientIdSetting.value}:${clientSecretSetting.value}`).toString('base64')
+      : undefined
+
+    const tokenRes = await fetch('https://api.twitter.com/2/oauth2/token', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        ...(authHeader ? { Authorization: authHeader } : {}),
+      },
+      body: tokenBody,
+    })
+
+    const tokenData = await tokenRes.json()
+    if (!tokenRes.ok || !tokenData.access_token) {
+      const msg = tokenData.error_description ?? tokenData.error ?? 'token_exchange_failed'
+      return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(msg)}`)
+    }
+
+    // Fetch authenticated user
+    const userRes = await fetch('https://api.twitter.com/2/users/me', {
+      headers: { Authorization: `Bearer ${tokenData.access_token}` },
+    })
+    const userData = await userRes.json()
+    const user = userData?.data
+
+    const expiresAt = tokenData.expires_in
+      ? String(Date.now() + tokenData.expires_in * 1000)
+      : String(Date.now() + 7200 * 1000)
+
+    await Promise.all([
+      prisma.setting.upsert({ where: { key: 'x_oauth_access_token' }, update: { value: tokenData.access_token }, create: { key: 'x_oauth_access_token', value: tokenData.access_token } }),
+      prisma.setting.upsert({ where: { key: 'x_oauth_token_expires_at' }, update: { value: expiresAt }, create: { key: 'x_oauth_token_expires_at', value: expiresAt } }),
+      tokenData.refresh_token && prisma.setting.upsert({ where: { key: 'x_oauth_refresh_token' }, update: { value: tokenData.refresh_token }, create: { key: 'x_oauth_refresh_token', value: tokenData.refresh_token } }),
+      user?.id && prisma.setting.upsert({ where: { key: 'x_oauth_user_id' }, update: { value: user.id }, create: { key: 'x_oauth_user_id', value: user.id } }),
+      user?.username && prisma.setting.upsert({ where: { key: 'x_oauth_username' }, update: { value: user.username }, create: { key: 'x_oauth_username', value: user.username } }),
+    ].filter(Boolean))
+
+    return NextResponse.redirect(`${IMPORT_URL}?x_connected=true`)
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : 'callback_error'
+    return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(msg)}`)
+  }
+}

app/api/import/x-oauth/disconnect/route.ts

@@ -0,0 +1,26 @@
+import { NextResponse } from 'next/server'
+import prisma from '@/lib/db'
+
+export async function POST() {
+  try {
+    await prisma.setting.deleteMany({
+      where: {
+        key: {
+          in: [
+            'x_oauth_access_token',
+            'x_oauth_refresh_token',
+            'x_oauth_user_id',
+            'x_oauth_username',
+            'x_oauth_token_expires_at',
+          ],
+        },
+      },
+    })
+    return NextResponse.json({ disconnected: true })
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : 'Failed to disconnect' },
+      { status: 500 },
+    )
+  }
+}

app/api/import/x-oauth/fetch/route.ts

@@ -0,0 +1,198 @@
+import { NextRequest, NextResponse } from 'next/server'
+import prisma from '@/lib/db'
+
+interface XMedia {
+  media_key: string
+  type: string
+  url?: string
+  preview_image_url?: string
+  variants?: Array<{ content_type?: string; bit_rate?: number; url: string }>
+}
+
+interface XUser { id: string; username: string; name: string }
+
+interface XTweet {
+  id: string
+  text: string
+  author_id?: string
+  created_at?: string
+  attachments?: { media_keys?: string[] }
+}
+
+function bestVideoUrl(variants: NonNullable<XMedia['variants']>): string | null {
+  return variants
+    .filter((v) => v.content_type === 'video/mp4' && v.url)
+    .sort((a, b) => (b.bit_rate ?? 0) - (a.bit_rate ?? 0))[0]?.url ?? null
+}
+
+function extractMedia(tweet: XTweet, mediaMap: Map<string, XMedia>) {
+  return (tweet.attachments?.media_keys ?? [])
+    .map((key) => {
+      const m = mediaMap.get(key)
+      if (!m) return null
+      if (m.type === 'photo') {
+        const url = m.url ?? m.preview_image_url ?? ''
+        return url ? { type: 'photo', url, thumbnailUrl: url } : null
+      }
+      if (m.type === 'video' || m.type === 'animated_gif') {
+        const url = m.variants ? (bestVideoUrl(m.variants) ?? m.preview_image_url ?? '') : (m.preview_image_url ?? '')
+        return url ? { type: m.type === 'animated_gif' ? 'gif' : 'video', url, thumbnailUrl: m.preview_image_url ?? url } : null
+      }
+      return null
+    })
+    .filter(Boolean) as { type: string; url: string; thumbnailUrl: string }[]
+}
+
+async function tryRefreshToken(
+  refreshToken: string,
+  clientId: string,
+  clientSecret?: string,
+): Promise<string | null> {
+  const body = new URLSearchParams({
+    grant_type: 'refresh_token',
+    refresh_token: refreshToken,
+    client_id: clientId,
+  })
+  const authHeader = clientSecret
+    ? 'Basic ' + Buffer.from(`${clientId}:${clientSecret}`).toString('base64')
+    : undefined
+  const res = await fetch('https://api.twitter.com/2/oauth2/token', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/x-www-form-urlencoded',
+      ...(authHeader ? { Authorization: authHeader } : {}),
+    },
+    body,
+  })
+  if (!res.ok) return null
+  const data = await res.json()
+  if (!data.access_token) return null
+
+  const expiresAt = data.expires_in
+    ? String(Date.now() + data.expires_in * 1000)
+    : String(Date.now() + 7200 * 1000)
+
+  await Promise.all([
+    prisma.setting.upsert({ where: { key: 'x_oauth_access_token' }, update: { value: data.access_token }, create: { key: 'x_oauth_access_token', value: data.access_token } }),
+    prisma.setting.upsert({ where: { key: 'x_oauth_token_expires_at' }, update: { value: expiresAt }, create: { key: 'x_oauth_token_expires_at', value: expiresAt } }),
+    data.refresh_token && prisma.setting.upsert({ where: { key: 'x_oauth_refresh_token' }, update: { value: data.refresh_token }, create: { key: 'x_oauth_refresh_token', value: data.refresh_token } }),
+  ].filter(Boolean))
+
+  return data.access_token
+}
+
+export async function POST(request: NextRequest): Promise<NextResponse> {
+  let maxPages = 10
+  try {
+    const body = await request.json()
+    if (typeof body.maxPages === 'number') maxPages = Math.min(body.maxPages, 50)
+  } catch { /* use default */ }
+
+  try {
+    const [accessTokenS, refreshTokenS, userIdS, clientIdS, clientSecretS, expiresAtS] = await Promise.all([
+      prisma.setting.findUnique({ where: { key: 'x_oauth_access_token' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_refresh_token' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_user_id' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_client_secret' } }),
+      prisma.setting.findUnique({ where: { key: 'x_oauth_token_expires_at' } }),
+    ])
+
+    if (!accessTokenS?.value || !userIdS?.value) {
+      return NextResponse.json({ error: 'Not connected to X. Please authenticate first.' }, { status: 401 })
+    }
+
+    let accessToken = accessTokenS.value
+    const userId = userIdS.value
+
+    // Auto-refresh if expired
+    if (expiresAtS?.value && Date.now() > parseInt(expiresAtS.value)) {
+      if (refreshTokenS?.value && clientIdS?.value) {
+        const newToken = await tryRefreshToken(refreshTokenS.value, clientIdS.value, clientSecretS?.value)
+        if (newToken) {
+          accessToken = newToken
+        } else {
+          return NextResponse.json(
+            { error: 'Token expired and refresh failed. Please reconnect your X account.' },
+            { status: 401 },
+          )
+        }
+      }
+    }
+
+    let imported = 0, skipped = 0, total = 0
+    let nextToken: string | undefined
+
+    for (let page = 0; page < maxPages; page++) {
+      const params = new URLSearchParams({
+        max_results: '100',
+        expansions: 'author_id,attachments.media_keys',
+        'tweet.fields': 'created_at,text,attachments,entities',
+        'user.fields': 'username,name',
+        'media.fields': 'url,type,variants,preview_image_url',
+        ...(nextToken ? { pagination_token: nextToken } : {}),
+      })
+
+      const res = await fetch(`https://api.twitter.com/2/users/${userId}/bookmarks?${params}`, {
+        headers: { Authorization: `Bearer ${accessToken}` },
+      })
+
+      if (!res.ok) {
+        const errText = await res.text()
+        return NextResponse.json(
+          { error: `X API ${res.status}: ${errText.slice(0, 200)}`, imported, skipped, total },
+          { status: 502 },
+        )
+      }
+
+      const data = await res.json()
+      if (!data.data?.length) break
+      total += data.data.length
+
+      const userMap = new Map<string, XUser>((data.includes?.users ?? []).map((u: XUser) => [u.id, u]))
+      const mediaMap = new Map<string, XMedia>((data.includes?.media ?? []).map((m: XMedia) => [m.media_key, m]))
+
+      for (const tweet of data.data as XTweet[]) {
+        const exists = await prisma.bookmark.findUnique({ where: { tweetId: tweet.id }, select: { id: true } })
+        if (exists) { skipped++; continue }
+
+        const author = tweet.author_id ? userMap.get(tweet.author_id) : undefined
+        const media = extractMedia(tweet, mediaMap)
+
+        const created = await prisma.bookmark.create({
+          data: {
+            tweetId: tweet.id,
+            text: tweet.text,
+            authorHandle: author?.username ?? 'unknown',
+            authorName: author?.name ?? 'Unknown',
+            tweetCreatedAt: tweet.created_at ? new Date(tweet.created_at) : null,
+            rawJson: JSON.stringify(tweet),
+            source: 'bookmark',
+          },
+        })
+
+        if (media.length > 0) {
+          await prisma.mediaItem.createMany({
+            data: media.map((m) => ({
+              bookmarkId: created.id,
+              type: m.type,
+              url: m.url,
+              thumbnailUrl: m.thumbnailUrl ?? null,
+            })),
+          })
+        }
+        imported++
+      }
+
+      nextToken = data.meta?.next_token
+      if (!nextToken) break
+    }
+
+    return NextResponse.json({ imported, skipped, total })
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : 'Fetch failed' },
+      { status: 500 },
+    )
+  }
+}