From b714021922e17705af5f557d681e74231face4a1 Mon Sep 17 00:00:00 2001 From: David Orban Date: Fri, 20 Mar 2026 13:07:29 +0100 Subject: [PATCH] feat(import): implement X OAuth 2.0 backend for live bookmark import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #50 The Live Import tab in the UI already had the full X OAuth 2.0 flow wired up on the frontend, but all five backend API routes were missing, causing the tab to display 'X OAuth not configured' and 'Could not connect to the server' for every user. This PR implements the missing backend: ## New routes - GET /api/import/x-oauth/status Returns { configured, connected, tokenExpired, user } so the UI can render the correct state. - GET /api/import/x-oauth/authorize Generates a PKCE code_verifier + code_challenge, stores the verifier keyed by a random state value, and returns the X OAuth 2.0 authorization URL for the client to redirect to. - GET /api/import/x-oauth/callback Exchanges the auth code + PKCE verifier for access/refresh tokens, fetches the authenticated user via /2/users/me, stores everything in the Setting table, then redirects to /import?x_connected=true. - POST /api/import/x-oauth/disconnect Removes all stored X OAuth tokens from the database. - POST /api/import/x-oauth/fetch Fetches bookmarks via the X API v2 /2/users/:id/bookmarks endpoint with full pagination support (up to 50 pages). Automatically refreshes an expired access token using the stored refresh token. Supports both photo and video/GIF media via media_keys expansion. ## Configuration Add `APP_URL` to your environment to override the callback URL base (defaults to `http://localhost:3000`). Useful when running behind a reverse proxy or on a custom port. ## Notes - Supports both public clients (PKCE only) and confidential clients (PKCE + Basic auth with client_id:client_secret) — autodetected from whether a Client Secret is configured in Settings. - PKCE verifiers are stored ephemerally in the Setting table and cleaned up immediately after the callback completes. - Token refresh is handled transparently in the fetch route. Co-Authored-By: Oz --- .env.example | 7 + app/api/import/x-oauth/authorize/route.ts | 48 +++++ app/api/import/x-oauth/callback/route.ts | 93 ++++++++++ app/api/import/x-oauth/disconnect/route.ts | 26 +++ app/api/import/x-oauth/fetch/route.ts | 198 +++++++++++++++++++++ app/api/import/x-oauth/status/route.ts | 30 ++++ 6 files changed, 402 insertions(+) create mode 100644 app/api/import/x-oauth/authorize/route.ts create mode 100644 app/api/import/x-oauth/callback/route.ts create mode 100644 app/api/import/x-oauth/disconnect/route.ts create mode 100644 app/api/import/x-oauth/fetch/route.ts create mode 100644 app/api/import/x-oauth/status/route.ts diff --git a/.env.example b/.env.example index 61d7e03..4638086 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,13 @@ DATABASE_URL="file:./prisma/dev.db" # Optional: custom API base URL (proxy or local model server) # ANTHROPIC_BASE_URL= +# ── X OAuth 2.0 (optional) ────────────────────────────────────────── + +# Base URL of your Siftly instance. Used to construct the OAuth callback URL. +# Required if running behind a reverse proxy or at a non-default port. +# Defaults to http://localhost:3000 if unset. +# APP_URL=http://localhost:3000 + # ── Access control (optional) ──────────────────────────────────────── # Set BOTH to enable HTTP Basic Auth on the entire app. diff --git a/app/api/import/x-oauth/authorize/route.ts b/app/api/import/x-oauth/authorize/route.ts new file mode 100644 index 0000000..e08d26b --- /dev/null +++ b/app/api/import/x-oauth/authorize/route.ts @@ -0,0 +1,48 @@ +import { NextResponse } from 'next/server' +import prisma from '@/lib/db' +import crypto from 'crypto' + +function base64url(buf: Buffer): string { + return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=/g, '') +} + +const APP_URL = process.env.APP_URL ?? 'http://localhost:3000' +const CALLBACK_URL = `${APP_URL}/api/import/x-oauth/callback` +const SCOPES = 'bookmark.read tweet.read users.read offline.access' + +export async function GET() { + try { + const clientId = await prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }) + if (!clientId?.value) { + return NextResponse.json({ error: 'X OAuth Client ID not configured in Settings' }, { status: 400 }) + } + + const codeVerifier = base64url(crypto.randomBytes(32)) + const codeChallenge = base64url(crypto.createHash('sha256').update(codeVerifier).digest()) + const state = base64url(crypto.randomBytes(16)) + + // Store PKCE verifier keyed by state (cleaned up after callback) + await prisma.setting.upsert({ + where: { key: `x_oauth_pkce_${state}` }, + update: { value: codeVerifier }, + create: { key: `x_oauth_pkce_${state}`, value: codeVerifier }, + }) + + const params = new URLSearchParams({ + response_type: 'code', + client_id: clientId.value, + redirect_uri: CALLBACK_URL, + scope: SCOPES, + state, + code_challenge: codeChallenge, + code_challenge_method: 'S256', + }) + + return NextResponse.json({ authUrl: `https://twitter.com/i/oauth2/authorize?${params}` }) + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : 'Failed to build auth URL' }, + { status: 500 }, + ) + } +} diff --git a/app/api/import/x-oauth/callback/route.ts b/app/api/import/x-oauth/callback/route.ts new file mode 100644 index 0000000..c9e347f --- /dev/null +++ b/app/api/import/x-oauth/callback/route.ts @@ -0,0 +1,93 @@ +import { NextRequest, NextResponse } from 'next/server' +import prisma from '@/lib/db' + +const APP_URL = process.env.APP_URL ?? 'http://localhost:3000' +const CALLBACK_URL = `${APP_URL}/api/import/x-oauth/callback` +const IMPORT_URL = `${APP_URL}/import` + +export async function GET(request: NextRequest): Promise { + const { searchParams } = new URL(request.url) + const code = searchParams.get('code') + const state = searchParams.get('state') + const error = searchParams.get('error') + + if (error) { + return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(error)}`) + } + + if (!code || !state) { + return NextResponse.redirect(`${IMPORT_URL}?x_error=missing_params`) + } + + try { + // Retrieve and clean up PKCE verifier + const verifierKey = `x_oauth_pkce_${state}` + const verifierSetting = await prisma.setting.findUnique({ where: { key: verifierKey } }) + if (!verifierSetting?.value) { + return NextResponse.redirect(`${IMPORT_URL}?x_error=invalid_state`) + } + const codeVerifier = verifierSetting.value + await prisma.setting.delete({ where: { key: verifierKey } }).catch(() => {}) + + const [clientIdSetting, clientSecretSetting] = await Promise.all([ + prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_client_secret' } }), + ]) + if (!clientIdSetting?.value) { + return NextResponse.redirect(`${IMPORT_URL}?x_error=client_id_missing`) + } + + // Exchange auth code for tokens + const tokenBody = new URLSearchParams({ + code, + grant_type: 'authorization_code', + client_id: clientIdSetting.value, + redirect_uri: CALLBACK_URL, + code_verifier: codeVerifier, + }) + + // Confidential clients send Basic auth; public clients omit it + const authHeader = clientSecretSetting?.value + ? 'Basic ' + Buffer.from(`${clientIdSetting.value}:${clientSecretSetting.value}`).toString('base64') + : undefined + + const tokenRes = await fetch('https://api.twitter.com/2/oauth2/token', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + ...(authHeader ? { Authorization: authHeader } : {}), + }, + body: tokenBody, + }) + + const tokenData = await tokenRes.json() + if (!tokenRes.ok || !tokenData.access_token) { + const msg = tokenData.error_description ?? tokenData.error ?? 'token_exchange_failed' + return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(msg)}`) + } + + // Fetch authenticated user + const userRes = await fetch('https://api.twitter.com/2/users/me', { + headers: { Authorization: `Bearer ${tokenData.access_token}` }, + }) + const userData = await userRes.json() + const user = userData?.data + + const expiresAt = tokenData.expires_in + ? String(Date.now() + tokenData.expires_in * 1000) + : String(Date.now() + 7200 * 1000) + + await Promise.all([ + prisma.setting.upsert({ where: { key: 'x_oauth_access_token' }, update: { value: tokenData.access_token }, create: { key: 'x_oauth_access_token', value: tokenData.access_token } }), + prisma.setting.upsert({ where: { key: 'x_oauth_token_expires_at' }, update: { value: expiresAt }, create: { key: 'x_oauth_token_expires_at', value: expiresAt } }), + tokenData.refresh_token && prisma.setting.upsert({ where: { key: 'x_oauth_refresh_token' }, update: { value: tokenData.refresh_token }, create: { key: 'x_oauth_refresh_token', value: tokenData.refresh_token } }), + user?.id && prisma.setting.upsert({ where: { key: 'x_oauth_user_id' }, update: { value: user.id }, create: { key: 'x_oauth_user_id', value: user.id } }), + user?.username && prisma.setting.upsert({ where: { key: 'x_oauth_username' }, update: { value: user.username }, create: { key: 'x_oauth_username', value: user.username } }), + ].filter(Boolean)) + + return NextResponse.redirect(`${IMPORT_URL}?x_connected=true`) + } catch (err) { + const msg = err instanceof Error ? err.message : 'callback_error' + return NextResponse.redirect(`${IMPORT_URL}?x_error=${encodeURIComponent(msg)}`) + } +} diff --git a/app/api/import/x-oauth/disconnect/route.ts b/app/api/import/x-oauth/disconnect/route.ts new file mode 100644 index 0000000..2a11d01 --- /dev/null +++ b/app/api/import/x-oauth/disconnect/route.ts @@ -0,0 +1,26 @@ +import { NextResponse } from 'next/server' +import prisma from '@/lib/db' + +export async function POST() { + try { + await prisma.setting.deleteMany({ + where: { + key: { + in: [ + 'x_oauth_access_token', + 'x_oauth_refresh_token', + 'x_oauth_user_id', + 'x_oauth_username', + 'x_oauth_token_expires_at', + ], + }, + }, + }) + return NextResponse.json({ disconnected: true }) + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : 'Failed to disconnect' }, + { status: 500 }, + ) + } +} diff --git a/app/api/import/x-oauth/fetch/route.ts b/app/api/import/x-oauth/fetch/route.ts new file mode 100644 index 0000000..dfb5bbe --- /dev/null +++ b/app/api/import/x-oauth/fetch/route.ts @@ -0,0 +1,198 @@ +import { NextRequest, NextResponse } from 'next/server' +import prisma from '@/lib/db' + +interface XMedia { + media_key: string + type: string + url?: string + preview_image_url?: string + variants?: Array<{ content_type?: string; bit_rate?: number; url: string }> +} + +interface XUser { id: string; username: string; name: string } + +interface XTweet { + id: string + text: string + author_id?: string + created_at?: string + attachments?: { media_keys?: string[] } +} + +function bestVideoUrl(variants: NonNullable): string | null { + return variants + .filter((v) => v.content_type === 'video/mp4' && v.url) + .sort((a, b) => (b.bit_rate ?? 0) - (a.bit_rate ?? 0))[0]?.url ?? null +} + +function extractMedia(tweet: XTweet, mediaMap: Map) { + return (tweet.attachments?.media_keys ?? []) + .map((key) => { + const m = mediaMap.get(key) + if (!m) return null + if (m.type === 'photo') { + const url = m.url ?? m.preview_image_url ?? '' + return url ? { type: 'photo', url, thumbnailUrl: url } : null + } + if (m.type === 'video' || m.type === 'animated_gif') { + const url = m.variants ? (bestVideoUrl(m.variants) ?? m.preview_image_url ?? '') : (m.preview_image_url ?? '') + return url ? { type: m.type === 'animated_gif' ? 'gif' : 'video', url, thumbnailUrl: m.preview_image_url ?? url } : null + } + return null + }) + .filter(Boolean) as { type: string; url: string; thumbnailUrl: string }[] +} + +async function tryRefreshToken( + refreshToken: string, + clientId: string, + clientSecret?: string, +): Promise { + const body = new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: refreshToken, + client_id: clientId, + }) + const authHeader = clientSecret + ? 'Basic ' + Buffer.from(`${clientId}:${clientSecret}`).toString('base64') + : undefined + const res = await fetch('https://api.twitter.com/2/oauth2/token', { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + ...(authHeader ? { Authorization: authHeader } : {}), + }, + body, + }) + if (!res.ok) return null + const data = await res.json() + if (!data.access_token) return null + + const expiresAt = data.expires_in + ? String(Date.now() + data.expires_in * 1000) + : String(Date.now() + 7200 * 1000) + + await Promise.all([ + prisma.setting.upsert({ where: { key: 'x_oauth_access_token' }, update: { value: data.access_token }, create: { key: 'x_oauth_access_token', value: data.access_token } }), + prisma.setting.upsert({ where: { key: 'x_oauth_token_expires_at' }, update: { value: expiresAt }, create: { key: 'x_oauth_token_expires_at', value: expiresAt } }), + data.refresh_token && prisma.setting.upsert({ where: { key: 'x_oauth_refresh_token' }, update: { value: data.refresh_token }, create: { key: 'x_oauth_refresh_token', value: data.refresh_token } }), + ].filter(Boolean)) + + return data.access_token +} + +export async function POST(request: NextRequest): Promise { + let maxPages = 10 + try { + const body = await request.json() + if (typeof body.maxPages === 'number') maxPages = Math.min(body.maxPages, 50) + } catch { /* use default */ } + + try { + const [accessTokenS, refreshTokenS, userIdS, clientIdS, clientSecretS, expiresAtS] = await Promise.all([ + prisma.setting.findUnique({ where: { key: 'x_oauth_access_token' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_refresh_token' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_user_id' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_client_secret' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_token_expires_at' } }), + ]) + + if (!accessTokenS?.value || !userIdS?.value) { + return NextResponse.json({ error: 'Not connected to X. Please authenticate first.' }, { status: 401 }) + } + + let accessToken = accessTokenS.value + const userId = userIdS.value + + // Auto-refresh if expired + if (expiresAtS?.value && Date.now() > parseInt(expiresAtS.value)) { + if (refreshTokenS?.value && clientIdS?.value) { + const newToken = await tryRefreshToken(refreshTokenS.value, clientIdS.value, clientSecretS?.value) + if (newToken) { + accessToken = newToken + } else { + return NextResponse.json( + { error: 'Token expired and refresh failed. Please reconnect your X account.' }, + { status: 401 }, + ) + } + } + } + + let imported = 0, skipped = 0, total = 0 + let nextToken: string | undefined + + for (let page = 0; page < maxPages; page++) { + const params = new URLSearchParams({ + max_results: '100', + expansions: 'author_id,attachments.media_keys', + 'tweet.fields': 'created_at,text,attachments,entities', + 'user.fields': 'username,name', + 'media.fields': 'url,type,variants,preview_image_url', + ...(nextToken ? { pagination_token: nextToken } : {}), + }) + + const res = await fetch(`https://api.twitter.com/2/users/${userId}/bookmarks?${params}`, { + headers: { Authorization: `Bearer ${accessToken}` }, + }) + + if (!res.ok) { + const errText = await res.text() + return NextResponse.json( + { error: `X API ${res.status}: ${errText.slice(0, 200)}`, imported, skipped, total }, + { status: 502 }, + ) + } + + const data = await res.json() + if (!data.data?.length) break + total += data.data.length + + const userMap = new Map((data.includes?.users ?? []).map((u: XUser) => [u.id, u])) + const mediaMap = new Map((data.includes?.media ?? []).map((m: XMedia) => [m.media_key, m])) + + for (const tweet of data.data as XTweet[]) { + const exists = await prisma.bookmark.findUnique({ where: { tweetId: tweet.id }, select: { id: true } }) + if (exists) { skipped++; continue } + + const author = tweet.author_id ? userMap.get(tweet.author_id) : undefined + const media = extractMedia(tweet, mediaMap) + + const created = await prisma.bookmark.create({ + data: { + tweetId: tweet.id, + text: tweet.text, + authorHandle: author?.username ?? 'unknown', + authorName: author?.name ?? 'Unknown', + tweetCreatedAt: tweet.created_at ? new Date(tweet.created_at) : null, + rawJson: JSON.stringify(tweet), + source: 'bookmark', + }, + }) + + if (media.length > 0) { + await prisma.mediaItem.createMany({ + data: media.map((m) => ({ + bookmarkId: created.id, + type: m.type, + url: m.url, + thumbnailUrl: m.thumbnailUrl ?? null, + })), + }) + } + imported++ + } + + nextToken = data.meta?.next_token + if (!nextToken) break + } + + return NextResponse.json({ imported, skipped, total }) + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : 'Fetch failed' }, + { status: 500 }, + ) + } +} diff --git a/app/api/import/x-oauth/status/route.ts b/app/api/import/x-oauth/status/route.ts new file mode 100644 index 0000000..2d43081 --- /dev/null +++ b/app/api/import/x-oauth/status/route.ts @@ -0,0 +1,30 @@ +import { NextResponse } from 'next/server' +import prisma from '@/lib/db' + +export async function GET() { + try { + const [clientId, accessToken, userId, username, expiresAt] = await Promise.all([ + prisma.setting.findUnique({ where: { key: 'x_oauth_client_id' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_access_token' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_user_id' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_username' } }), + prisma.setting.findUnique({ where: { key: 'x_oauth_token_expires_at' } }), + ]) + + const configured = !!clientId?.value + const connected = !!(accessToken?.value && userId?.value) + const tokenExpired = expiresAt?.value ? Date.now() > parseInt(expiresAt.value) : false + + return NextResponse.json({ + configured, + connected, + tokenExpired: connected ? tokenExpired : undefined, + user: connected ? { id: userId?.value, username: username?.value } : null, + }) + } catch (err) { + return NextResponse.json( + { error: err instanceof Error ? err.message : 'Failed' }, + { status: 500 }, + ) + } +}