From 1273192bc06e60c7762fde9e623096f244f540f8 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 11:36:43 -0700 Subject: [PATCH 01/44] ACHOO-119: local caching --- .gitignore | 5 +- app/api/acquia/applications/route.ts | 67 ++++----- app/api/acquia/views/route.ts | 86 ++++++------ app/api/acquia/visits/route.ts | 83 ++++++----- app/api/cache/route.ts | 27 ++++ app/api/revalidate/route.ts | 26 ++++ app/applications/[uuid]/page.tsx | 99 +++++++++++-- components/Dashboard.tsx | 200 +++++++++++++++++++++------ lib/acquia-api.ts | 119 ++-------------- lib/cache.ts | 103 ++++++++++++++ vercel.json | 13 ++ 11 files changed, 545 insertions(+), 283 deletions(-) create mode 100644 app/api/cache/route.ts create mode 100644 app/api/revalidate/route.ts create mode 100644 lib/cache.ts create mode 100644 vercel.json diff --git a/.gitignore b/.gitignore index 578c7c4..c06d162 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,7 @@ next-env.d.ts # saml saml-sp.key -saml-sp.crt \ No newline at end of file +saml-sp.crt + +# Cache directory +.cache/ \ No newline at end of file diff --git a/app/api/acquia/applications/route.ts b/app/api/acquia/applications/route.ts index 808b927..1308110 100644 --- a/app/api/acquia/applications/route.ts +++ b/app/api/acquia/applications/route.ts @@ -2,54 +2,43 @@ import { NextRequest, NextResponse } from 'next/server'; import AcquiaApiServiceFixed from '@/lib/acquia-api'; export async function GET(request: NextRequest) { - // console.log('🚀 Applications API Route called'); + try { + const { searchParams } = new URL(request.url); + const subscriptionUuid = searchParams.get('subscriptionUuid'); - // Update the API service initialization with better error handling - if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { - console.error('❌ Missing required environment variables!'); - console.error('Available env vars:', Object.keys(process.env).filter(k => k.startsWith('ACQUIA'))); - return NextResponse.json( - { - error: 'Server configuration error: missing API credentials', - envCheck: { - ACQUIA_API_KEY: process.env.ACQUIA_API_KEY ? `${process.env.ACQUIA_API_KEY.substring(0, 8)}...` : 'missing', - ACQUIA_API_SECRET: process.env.ACQUIA_API_SECRET ? 'present' : 'missing', - ACQUIA_API_BASE_URL: process.env.ACQUIA_API_BASE_URL || 'missing', - ACQUIA_AUTH_BASE_URL: process.env.ACQUIA_AUTH_BASE_URL || 'missing' - } - }, - { status: 500 } - ); - } + console.log('🔍 Applications API called with subscriptionUuid:', subscriptionUuid); - try { - // Update the API service initialization - const apiService = new AcquiaApiServiceFixed({ + if (!subscriptionUuid) { + return NextResponse.json({ error: 'subscriptionUuid is required' }, { status: 400 }); + } + + // Check environment variables + if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { + console.error('❌ Missing API credentials'); + return NextResponse.json({ error: 'Missing API credentials' }, { status: 500 }); + } + + const service = new AcquiaApiServiceFixed({ baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', - apiKey: process.env.ACQUIA_API_KEY!, - apiSecret: process.env.ACQUIA_API_SECRET!, + apiKey: process.env.ACQUIA_API_KEY, + apiSecret: process.env.ACQUIA_API_SECRET, }); - // console.log('🔧 Using FIXED API Service for applications'); + console.log('🔧 Fetching applications...'); + const applications = await service.getApplications(); + console.log('✅ Got applications:', applications.length); + + const response = NextResponse.json(applications); + response.headers.set('Cache-Control', 'public, s-maxage=21600, stale-while-revalidate=3600'); - const applications = await apiService.getApplications(); - - // console.log('✅ Successfully fetched applications data, count:', applications.length); - - return NextResponse.json(applications); + return response; } catch (error) { - console.error('❌ API Route Error:', error); - - if (error instanceof Error) { - console.error('🔍 Error name:', error.name); - console.error('🔍 Error message:', error.message); - console.error('🔍 Error stack:', error.stack); - } - + console.error('❌ Applications API Error:', error); + console.error('❌ Error stack:', error instanceof Error ? error.stack : 'No stack trace'); return NextResponse.json( - { - error: 'Failed to fetch applications data', + { + error: 'Failed to fetch applications', details: error instanceof Error ? error.message : 'Unknown error' }, { status: 500 } diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 0975826..9dd74fb 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -1,43 +1,44 @@ import { NextRequest, NextResponse } from 'next/server'; import AcquiaApiServiceFixed from '@/lib/acquia-api'; +import { getCachedData, setCachedData, generateApiCacheKey } from '@/lib/cache'; + +export const revalidate = 21600; // 6 hours cache at route level export async function GET(request: NextRequest) { const searchParams = request.nextUrl.searchParams; const subscriptionUuid = searchParams.get('subscriptionUuid'); const from = searchParams.get('from'); const to = searchParams.get('to'); - const resolution = searchParams.get('resolution'); // Get granularity for daily data - /** - console.log('🚀 Views by Application API Route called with params:', { - subscriptionUuid, - from, - to, - resolution - }); - */ + const resolution = searchParams.get('resolution'); + + console.log('🔍 Views API called with params:', { subscriptionUuid, from, to, resolution }); + if (!subscriptionUuid) { - console.error('❌ Missing required parameter: subscriptionUuid'); return NextResponse.json( { error: 'subscriptionUuid is required' }, { status: 400 } ); } - if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { - console.error('❌ Missing required environment variables!'); - console.error('Available env vars:', Object.keys(process.env).filter(k => k.startsWith('ACQUIA'))); - return NextResponse.json( - { - error: 'Server configuration error: missing API credentials', - envCheck: { - ACQUIA_API_KEY: process.env.ACQUIA_API_KEY ? `${process.env.ACQUIA_API_KEY.substring(0, 8)}...` : 'missing', - ACQUIA_API_SECRET: process.env.ACQUIA_API_SECRET ? 'present' : 'missing', - ACQUIA_API_BASE_URL: process.env.ACQUIA_API_BASE_URL || 'missing', - ACQUIA_AUTH_BASE_URL: process.env.ACQUIA_AUTH_BASE_URL || 'missing' - } - }, - { status: 500 } - ); + // Generate cache key + const cacheKey = generateApiCacheKey('views', { + subscriptionUuid, + from: from || 'no-from', + to: to || 'no-to', + resolution: resolution || 'no-resolution' + }); + + console.log('🗝️ Views cache key:', cacheKey); + + // Check cache first + const cachedResult = await getCachedData(cacheKey); + if (cachedResult) { + console.log('📦 Returning cached views data'); + return NextResponse.json({ + ...cachedResult, + cached: true, + cacheKey + }); } try { @@ -48,10 +49,7 @@ export async function GET(request: NextRequest) { apiSecret: process.env.ACQUIA_API_SECRET!, }); - apiService.setProgressCallback((progress) => { - // console.log('📈 Views progress:', progress); - }); - + console.log('🔧 Fetching fresh views data from API...'); const data = await apiService.getViewsDataByApplication( subscriptionUuid, from || undefined, @@ -59,25 +57,31 @@ export async function GET(request: NextRequest) { resolution || undefined ); - // console.log('✅ Successfully fetched ALL views by application data, total count:', data.length); + console.log('✅ Got fresh views data:', data.length); - return NextResponse.json({ + const result = { data, totalItems: data.length, - message: `Successfully fetched ${data.length} view records across all pages`, - }); + message: `Successfully fetched ${data.length} view records`, + cached: false, + timestamp: new Date().toISOString(), + cacheKey + }; + + // Cache the result + await setCachedData(cacheKey, result); + + const response = NextResponse.json(result); + response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); + + return response; } catch (error) { - console.error('❌ API Route Error:', error); - if (error instanceof Error) { - console.error('🔍 Error name:', error.name); - console.error('🔍 Error message:', error.message); - console.error('🔍 Error stack:', error.stack); - } + console.error('❌ Views API Error:', error); return NextResponse.json( { - error: 'Failed to fetch views by application data', - details: error instanceof Error ? error.message : 'Unknown error', + error: 'Failed to fetch views data', + details: error instanceof Error ? error.message : 'Unknown error' }, { status: 500 } ); diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index 94104ae..ae525c9 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -1,60 +1,53 @@ import { NextRequest, NextResponse } from 'next/server'; import AcquiaApiServiceFixed from '@/lib/acquia-api'; +import { getCachedData, setCachedData, generateApiCacheKey } from '@/lib/cache'; export async function GET(request: NextRequest) { const searchParams = request.nextUrl.searchParams; const subscriptionUuid = searchParams.get('subscriptionUuid'); const from = searchParams.get('from'); const to = searchParams.get('to'); - const resolution = searchParams.get('resolution'); // Get granularity for daily data + const resolution = searchParams.get('resolution'); - /** - console.log('🚀 Visits by Application API Route called with params:', { - subscriptionUuid, - from, - to - }); - */ + console.log('🔍 Visits API called with params:', { subscriptionUuid, from, to, resolution }); if (!subscriptionUuid) { - console.error('❌ Missing required parameter: subscriptionUuid'); return NextResponse.json( { error: 'subscriptionUuid is required' }, { status: 400 } ); } - // Update the API service initialization with better error handling - if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { - console.error('❌ Missing required environment variables!'); - console.error('Available env vars:', Object.keys(process.env).filter(k => k.startsWith('ACQUIA'))); - return NextResponse.json( - { - error: 'Server configuration error: missing API credentials', - envCheck: { - ACQUIA_API_KEY: process.env.ACQUIA_API_KEY ? `${process.env.ACQUIA_API_KEY.substring(0, 8)}...` : 'missing', - ACQUIA_API_SECRET: process.env.ACQUIA_API_SECRET ? 'present' : 'missing', - ACQUIA_API_BASE_URL: process.env.ACQUIA_API_BASE_URL || 'missing', - ACQUIA_AUTH_BASE_URL: process.env.ACQUIA_AUTH_BASE_URL || 'missing' - } - }, - { status: 500 } - ); + // Generate cache key + const cacheKey = generateApiCacheKey('visits', { + subscriptionUuid, + from: from || 'no-from', + to: to || 'no-to', + resolution: resolution || 'no-resolution' + }); + + console.log('🗝️ Visits cache key:', cacheKey); + + // Check cache first + const cachedResult = await getCachedData(cacheKey); + if (cachedResult) { + console.log('📦 Returning cached visits data'); + return NextResponse.json({ + ...cachedResult, + cached: true, + cacheKey + }); } try { const apiService = new AcquiaApiServiceFixed({ baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', - apiKey: process.env.ACQUIA_API_KEY, - apiSecret: process.env.ACQUIA_API_SECRET, - }); - - apiService.setProgressCallback((progress) => { - // console.log('📊 Visits progress:', progress); + apiKey: process.env.ACQUIA_API_KEY!, + apiSecret: process.env.ACQUIA_API_SECRET!, }); - // console.log('🔧 Using FIXED API Service for visits by application (with pagination)'); + console.log('🔧 Fetching fresh visits data from API...'); const data = await apiService.getVisitsDataByApplication( subscriptionUuid, from || undefined, @@ -62,20 +55,26 @@ export async function GET(request: NextRequest) { resolution || undefined ); - // console.log('✅ Successfully fetched ALL visits by application data, total count:', data.length); + console.log('✅ Got fresh visits data:', data.length); - return NextResponse.json({ + const result = { data, totalItems: data.length, - message: `Successfully fetched ${data.length} visit records across all pages` - }); + message: `Successfully fetched ${data.length} visit records across all pages`, + cached: false, + timestamp: new Date().toISOString(), + cacheKey + }; + + // Cache the result + await setCachedData(cacheKey, result); + + const response = NextResponse.json(result); + response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); + + return response; } catch (error) { - console.error('❌ API Route Error:', error); - if (error instanceof Error) { - console.error('🔍 Error name:', error.name); - console.error('🔍 Error message:', error.message); - console.error('🔍 Error stack:', error.stack); - } + console.error('❌ Visits API Error:', error); return NextResponse.json( { diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts new file mode 100644 index 0000000..f99358f --- /dev/null +++ b/app/api/cache/route.ts @@ -0,0 +1,27 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { clearAllCache } from '@/lib/cache'; + +export async function DELETE(request: NextRequest) { + try { + await clearAllCache(); + return NextResponse.json({ + message: 'Cache cleared successfully', + timestamp: new Date().toISOString() + }); + } catch (error) { + console.error('Cache clear error:', error); + return NextResponse.json( + { error: 'Failed to clear cache' }, + { status: 500 } + ); + } +} + +export async function GET(request: NextRequest) { + return NextResponse.json({ + message: 'Cache management API', + endpoints: { + 'DELETE /api/cache': 'Clear all cached data' + } + }); +} \ No newline at end of file diff --git a/app/api/revalidate/route.ts b/app/api/revalidate/route.ts new file mode 100644 index 0000000..67cfcf9 --- /dev/null +++ b/app/api/revalidate/route.ts @@ -0,0 +1,26 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { revalidateTag } from 'next/cache'; + +export async function POST(request: NextRequest) { + try { + const { tags } = await request.json(); + + if (!Array.isArray(tags)) { + return NextResponse.json({ error: 'tags must be an array' }, { status: 400 }); + } + + // Revalidate specified cache tags + for (const tag of tags) { + revalidateTag(tag); + } + + return NextResponse.json({ + message: `Revalidated tags: ${tags.join(', ')}`, + revalidated: true, + now: Date.now() + }); + } catch (error) { + console.error('Revalidation error:', error); + return NextResponse.json({ error: 'Failed to revalidate' }, { status: 500 }); + } +} \ No newline at end of file diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 339d86d..9e7e0c1 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -24,8 +24,8 @@ interface AcquiaApiResponse { // Use 'any' in the signature to satisfy the Next.js build process for client components. export default function ApplicationDetailPage({ params }: any) { - // Re-introduce the type inside the component for type safety. - const typedParams: { uuid: string } = params; + // Use a single stable uuid primitive from params + const { uuid } = params; const [subscriptionUuid, setSubscriptionUuid] = useState(DEFAULT_SUBSCRIPTION_UUID); const [from, setFrom] = useState(''); @@ -45,20 +45,33 @@ export default function ApplicationDetailPage({ params }: any) { // Fetch application name on mount or when subscriptionUuid changes useEffect(() => { const fetchAppName = async () => { + if (!subscriptionUuid) return; + try { setLoadingStep('Fetching application info...'); const res = await fetch(`/api/acquia/applications?subscriptionUuid=${subscriptionUuid}`); + if (!res.ok) { + console.error('applications API responded with non-OK status', res.status); + setAppName(''); + return; + } const apps = await res.json(); - const app = Array.isArray(apps) ? apps.find((a: any) => a.uuid === typedParams.uuid) : null; + console.debug('fetchAppName', { subscriptionUuid, uuid, appsLength: apps?.length ?? 0 }); + + // apps is now the array directly, not wrapped in _embedded + const app = Array.isArray(apps) ? apps.find((a: any) => a.uuid === uuid) : null; + console.debug('found app:', { found: !!app, appName: app?.name }); setAppName(app ? app.name : ''); - } catch { + } catch (error) { + console.error('Error fetching app name:', error); setAppName(''); } finally { setLoadingStep(''); } }; - if (subscriptionUuid) fetchAppName(); - }, [subscriptionUuid, typedParams.uuid]); + + fetchAppName(); + }, [subscriptionUuid, uuid]); const fetchAppDetail = async () => { setLoading(true); @@ -85,12 +98,20 @@ export default function ApplicationDetailPage({ params }: any) { dailyVisitsRes.ok ? dailyVisitsRes.json() : {}, ]); + setLoadingStep('Processing data...'); + + console.log('📊 Application detail - processing data for UUID:', uuid); + console.log('📊 Visits data length:', dailyVisitsRaw.data?.length); + console.log('📈 Views data length:', dailyViewsRaw.data?.length); + // Helper to process and aggregate daily data with proper types const processDailyData = (rawData: AcquiaApiResponse, metric: 'views' | 'visits'): DailyDataPoint[] => { const dailyMap = new Map(); const dataArray = rawData.data || []; - const appData = dataArray.filter((d) => d.applicationUuid === typedParams.uuid); + // Filter for this specific application using the uuid from params + const appData = dataArray.filter((d) => d.applicationUuid === uuid); + console.log(`📊 Processing ${metric} for app ${uuid}: found ${appData.length} records`); for (const record of appData) { const date = record.date.split('T')[0]; @@ -98,9 +119,12 @@ export default function ApplicationDetailPage({ params }: any) { dailyMap.set(date, (dailyMap.get(date) || 0) + value); } - return Array.from(dailyMap.entries()) + const result = Array.from(dailyMap.entries()) .map(([date, value]) => ({ date, value })) .sort((a, b) => a.date.localeCompare(b.date)); + + console.log(`📊 Daily ${metric} data points:`, result.length); + return result; }; const processedDailyViews = processDailyData(dailyViewsRaw, 'views'); @@ -138,7 +162,7 @@ export default function ApplicationDetailPage({ params }: any) { return (

- Views and Visits Data for {appName ? appName : {typedParams.uuid}} + Views and Visits Data for {appName ? appName : {uuid}}

@@ -220,14 +244,14 @@ export default function ApplicationDetailPage({ params }: any) {
{error}
)} {!appName && !loading ? ( -
No application found with UUID: {typedParams.uuid}
+
No application found with UUID: {uuid}
) : (
Name: {appName}
- UUID: {typedParams.uuid} + UUID: {uuid}
Views{from && to ? ` (${from} to ${to})` : ''}: {views.toLocaleString()} ({viewsPct.toFixed(1)}%) @@ -279,6 +303,59 @@ export default function ApplicationDetailPage({ params }: any) {
)} + + {/* Debug Info Section - only in development */} + {process.env.NODE_ENV === 'development' && ( +
+

Debug Info

+ 
+            {JSON.stringify({
+              uuid,
+              subscriptionUuid,
+              appName,
+              visitsDataLength: dailyVisits?.length || 0,
+              viewsDataLength: dailyViews?.length || 0,
+              totalViews: views,
+              totalVisits: visits,
+              hasError: !!error,
+              loading,
+              from,
+              to,
+              cacheInfo: {
+                visitsUrl: `/api/acquia/visits?subscriptionUuid=${subscriptionUuid}&from=${from}&to=${to}&resolution=day`,
+                viewsUrl: `/api/acquia/views?subscriptionUuid=${subscriptionUuid}&from=${from}&to=${to}&resolution=day`,
+              }
+            }, null, 2)}
+
+
+ + +
+
+ )}
); } \ No newline at end of file diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 1857427..2bf01c2 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -38,17 +38,21 @@ const Dashboard: React.FC = () => { const [applications, setApplications] = useState([]); const [applicationMap, setApplicationMap] = useState>({}); const [activeTab, setActiveTab] = useState(TABS[0].key); + const [cacheClearing, setCacheClearing] = useState(false); const fetchApplications = async () => { + if (!subscriptionUuid) return; + try { - const response = await fetch('/api/acquia/applications'); + console.log('📱 Fetching applications for subscription:', subscriptionUuid); + const response = await fetch(`/api/acquia/applications?subscriptionUuid=${subscriptionUuid}`); if (!response.ok) { - console.error('Failed to fetch applications'); + console.error('Failed to fetch applications:', response.status); return; } const apps = await response.json(); - // console.log('📱 Fetched applications:', apps.length); + console.log('📱 Fetched applications:', apps.length, apps); setApplications(apps); @@ -59,7 +63,7 @@ const Dashboard: React.FC = () => { }); setApplicationMap(appMap); - // console.log('📱 Created application map with', Object.keys(appMap).length, 'entries'); + console.log('📱 Created application map:', appMap); } catch (error) { console.error('Error fetching applications:', error); @@ -79,8 +83,10 @@ const Dashboard: React.FC = () => { }; useEffect(() => { - fetchApplications(); - }, []); + if (subscriptionUuid) { + fetchApplications(); + } + }, [subscriptionUuid]); const fetchData = async () => { if (!subscriptionUuid) { @@ -99,17 +105,29 @@ const Dashboard: React.FC = () => { const startTime = Date.now(); try { + // Ensure we have applications first + if (Object.keys(applicationMap).length === 0) { + setLoadingStep('Fetching applications...'); + await fetchApplications(); + } + const params = new URLSearchParams({ subscriptionUuid, ...(dateFrom && { from: dateFrom }), ...(dateTo && { to: dateTo }), }); - // console.log('🔄 Fetching data with params:', { subscriptionUuid, dateFrom, dateTo }); + // Add cache-friendly headers to requests + const fetchOptions = { + headers: { + 'Cache-Control': 'public, max-age=21600', + }, + }; // Fetch visits data setLoadingStep('Fetching visits data from Acquia API...'); - const visitsResponse = await fetch(`/api/acquia/visits?${params}`); + console.log('📊 Fetching visits with URL:', `/api/acquia/visits?${params}`); + const visitsResponse = await fetch(`/api/acquia/visits?${params}`, fetchOptions); if (!visitsResponse.ok) { const visitsError = await visitsResponse.text(); @@ -118,10 +136,16 @@ const Dashboard: React.FC = () => { } const visitsResult = await visitsResponse.json(); - // console.log('📊 Received visits result with length:', Array.isArray(visitsResult) ? visitsResult.length : 'not an array'); + console.log('📊 Visits API response:', { + dataLength: visitsResult.data?.length, + cached: visitsResult.cached, + timestamp: visitsResult.timestamp + }); + // Fetch views data setLoadingStep('Fetching views data from Acquia API...'); - const viewsResponse = await fetch(`/api/acquia/views?${params}`); + console.log('📈 Fetching views with URL:', `/api/acquia/views?${params}`); + const viewsResponse = await fetch(`/api/acquia/views?${params}`, fetchOptions); if (!viewsResponse.ok) { const viewsError = await viewsResponse.text(); @@ -130,26 +154,47 @@ const Dashboard: React.FC = () => { } const viewsResult = await viewsResponse.json(); - // console.log('📈 Received views result with length:', Array.isArray(viewsResult) ? viewsResult.length : 'not an array'); + console.log('📈 Views API response:', { + dataLength: viewsResult.data?.length, + cached: viewsResult.cached, + timestamp: viewsResult.timestamp + }); + setLoadingStep('Processing data...'); - // Handle different response formats - const visitsArray = Array.isArray(visitsResult) ? visitsResult : - Array.isArray(visitsResult.data) ? visitsResult.data : []; + // Both APIs return { data: [...], totalItems: number, message: string } + const visitsArray = visitsResult.data || []; + const viewsArray = viewsResult.data || []; - const viewsArray = Array.isArray(viewsResult) ? viewsResult : - Array.isArray(viewsResult.data) ? viewsResult.data : []; + console.log('📊 Raw visits array length:', visitsArray.length); + console.log('📈 Raw views array length:', viewsArray.length); + console.log('📱 Current application map size:', Object.keys(applicationMap).length); // Add application names to the data - const visitsWithNames = visitsArray.map((visit: { applicationUuid: string; applicationName: any; }) => ({ - ...visit, - applicationName: applicationMap[visit.applicationUuid] || visit.applicationName || (visit.applicationUuid ? `App ${visit.applicationUuid.substring(0, 8)}` : 'Unknown App') - })); + const visitsWithNames = visitsArray.map((visit: any) => { + const appName = applicationMap[visit.applicationUuid] || + visit.applicationName || + (visit.applicationUuid ? `App ${visit.applicationUuid.substring(0, 8)}` : 'Unknown App'); + + return { + ...visit, + applicationName: appName + }; + }); - const viewsWithNames = viewsArray.map((view: { applicationUuid: string; applicationName: any; }) => ({ - ...view, - applicationName: applicationMap[view.applicationUuid] || view.applicationName || (view.applicationUuid ? `App ${view.applicationUuid.substring(0, 8)}` : 'Unknown App') - })); + const viewsWithNames = viewsArray.map((view: any) => { + const appName = applicationMap[view.applicationUuid] || + view.applicationName || + (view.applicationUuid ? `App ${view.applicationUuid.substring(0, 8)}` : 'Unknown App'); + + return { + ...view, + applicationName: appName + }; + }); + + console.log('📊 Final visits data length:', visitsWithNames.length); + console.log('📈 Final views data length:', viewsWithNames.length); setVisitsData(visitsWithNames); setViewsData(viewsWithNames); @@ -163,7 +208,7 @@ const Dashboard: React.FC = () => { } catch (err) { console.error('❌ Dashboard fetch error:', err); const errorMessage = err instanceof Error ? err.message : 'An error occurred while fetching data'; - setError(errorMessage); + setError(errorMessage); } finally { const endTime = Date.now(); const timeElapsed = (endTime - startTime) / 1000; @@ -213,9 +258,32 @@ const Dashboard: React.FC = () => { .sort((a, b) => b.views - a.views); }, [viewsData, applicationMap]); + const clearCache = async () => { + setCacheClearing(true); + try { + const response = await fetch('/api/revalidate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + tags: ['applications', 'views', 'visits', subscriptionUuid] + }) + }); + + if (response.ok) { + alert('Cache cleared successfully!'); + } else { + alert('Failed to clear cache'); + } + } catch (error) { + console.error('Cache clearing error:', error); + alert('Error clearing cache'); + } finally { + setCacheClearing(false); + } + }; + return ( -
+
This dashboard shows your monthly usage for Acquia Cloud hosting.
@@ -278,23 +346,33 @@ const Dashboard: React.FC = () => { > {loading ? 'Fetching Data...' : 'Fetch Analytics Data'} - {loading && ( -
- -
{loadingStep}
-
- )} - - {!loading && elapsedTime !== null && ( -
- -
- Data loaded in {elapsedTime.toFixed(1)} seconds -
-
- )} + +
+ {loading && ( +
+ +
{loadingStep}
+
+ )} + + {!loading && elapsedTime !== null && ( +
+ +
+ Data loaded in {elapsedTime.toFixed(1)} seconds +
+
+ )} +

(Note that it can take several minutes to fetch data from the Acquia API.)

@@ -385,6 +463,44 @@ const Dashboard: React.FC = () => { )}
{/* ...loading and error messages... */} + + {/* Debug Info - Only visible in development mode */} + {process.env.NODE_ENV === 'development' && ( +
+

Debug Info

+ 
+            {JSON.stringify({
+              subscriptionUuid,
+              applicationMapSize: Object.keys(applicationMap || {}).length,
+              applicationsLength: applications?.length || 0,
+              visitsDataLength: visitsData?.length || 0,
+              viewsDataLength: viewsData?.length || 0,
+              hasError: !!error,
+              loading,
+              dateFrom,
+              dateTo,
+              fetchStats,
+              cacheInfo: {
+                requestUrl: `/api/acquia/visits?subscriptionUuid=${subscriptionUuid}&from=${dateFrom}&to=${dateTo}`,
+                requestCount: Math.floor(Date.now() / 1000) % 100, // Simple request counter
+              }
+            }, null, 2)}
+
+ +
+ )}
); diff --git a/lib/acquia-api.ts b/lib/acquia-api.ts index 5278f0f..52c7940 100644 --- a/lib/acquia-api.ts +++ b/lib/acquia-api.ts @@ -1,21 +1,11 @@ import axios from 'axios'; -// --- Caching Infrastructure --- -interface CacheEntry { - data: T; - timestamp: number; // The time the data was cached -} - -const cache: Record> = {}; -const CACHE_DURATION_MS = 6 * 60 * 60 * 1000; // 6 hours in milliseconds - /** * Generates a unique string key for caching based on the request parameters. */ const generateCacheKey = (parts: (string | undefined | null)[]): string => { return parts.filter(Boolean).map(part => encodeURIComponent(part as string)).join(':'); }; -// --- End Caching Infrastructure --- export interface VisitsData { applicationUuid: string; @@ -136,7 +126,7 @@ class AcquiaApiServiceFixed { const authMethods = [ // Method 1: Basic Auth async () => { - // console.log('🔐 Trying Basic Auth method...'); + console.log('🔐 Trying Basic Auth method...'); const credentials = Buffer.from(`${cleanApiKey}:${cleanApiSecret}`).toString('base64'); const response = await axios({ method: 'POST', @@ -160,7 +150,7 @@ class AcquiaApiServiceFixed { // Method 2: Form parameters async () => { - // console.log('🔐 Trying Form Parameters method...'); + console.log('🔐 Trying Form Parameters method...'); const formData = new URLSearchParams(); formData.append('grant_type', 'client_credentials'); formData.append('client_id', cleanApiKey); @@ -187,7 +177,7 @@ class AcquiaApiServiceFixed { // Method 3: Use correct client ID format (if UUID is in different format) async () => { - // console.log('🔐 Trying with alternate client ID format...'); + console.log('🔐 Trying with alternate client ID format...'); // Try with a UUID format if the key is not already in UUID format const isUuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(cleanApiKey); @@ -226,7 +216,7 @@ class AcquiaApiServiceFixed { try { const token = await method(); this.accessToken = token; - // console.log('✅ Successfully authenticated!'); + console.log('✅ Successfully authenticated!'); return token; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); @@ -275,20 +265,11 @@ class AcquiaApiServiceFixed { } async getApplications(): Promise { - const cacheKey = 'applications'; - const cachedEntry = cache[cacheKey]; - - if (cachedEntry && (Date.now() - cachedEntry.timestamp < CACHE_DURATION_MS)) { - // console.log('✅ Returning cached applications data.'); - return cachedEntry.data; - } - + // Temporarily disable caching to debug try { - // console.log(`🔍 Fetching all applications (cache miss or stale)`); - + console.log(`🔍 Fetching all applications from API`); const response = await this.makeAuthenticatedRequest('/applications'); - - // console.log('✅ Applications API Response Status:', response.status); + console.log('✅ Applications API Response Status:', response.status); let applications: Application[] = []; @@ -306,14 +287,11 @@ class AcquiaApiServiceFixed { })) })); - // console.log(`✅ Extracted ${applications.length} applications`); + console.log(`✅ Extracted ${applications.length} applications`); } else { console.warn('⚠️ No applications found in response'); } - // Store the fresh data in the cache - cache[cacheKey] = { data: applications, timestamp: Date.now() }; - return applications; } catch (error) { console.error('❌ Error fetching applications:', error); @@ -564,18 +542,7 @@ class AcquiaApiServiceFixed { to?: string, resolution?: string ): Promise { - const cacheKey = generateCacheKey([baseEndpoint, subscriptionUuid, from, to, resolution]); - const cachedEntry = cache[cacheKey]; - - if (cachedEntry && (Date.now() - cachedEntry.timestamp < CACHE_DURATION_MS)) { - // console.log(`✅ Returning cached ${dataType} data for key: ${cacheKey}`); - this.reportProgress({ - step: `Using cached ${dataType} data.`, - itemsCollected: cachedEntry.data.length - }); - return cachedEntry.data; - } - + // Temporarily disable caching to debug let allData: T[] = []; let currentPage = 1; let totalPages = 1; @@ -586,57 +553,26 @@ class AcquiaApiServiceFixed { while (hasMorePages) { try { const params = new URLSearchParams(); - // Add filter parameter if we have date range if (filterParam) { params.append('filter', filterParam); } - - // Use the resolution parameter if it exists if (resolution) { params.append('resolution', resolution); - // console.log(`📊 Using resolution: ${resolution}`); } - if (currentPage > 1) { params.append('page', currentPage.toString()); } const fullEndpoint = `${baseEndpoint}?${params.toString()}`; - this.reportProgress({ - step: `Fetching ${dataType} data (page ${currentPage})...`, - currentPage, - totalPages: totalPages > 1 ? totalPages : undefined, - itemsCollected: allData.length - }); + console.log(`🔍 Fetching ${dataType} page ${currentPage}:`, fullEndpoint); const startTime = Date.now(); const response = await this.makeAuthenticatedRequest(fullEndpoint); const endTime = Date.now(); - // console.log(`✅ Request completed in ${endTime - startTime}ms`); - // console.log(`📊 Response status: ${response.status}`); - - // Log some response details to debug date issues - if (response.data._embedded?.items?.length > 0) { - const firstItem = response.data._embedded.items[0]; - if (firstItem.datapoints?.length > 0) { - const firstDatapoint = firstItem.datapoints[0]; - const lastDatapoint = firstItem.datapoints[firstItem.datapoints.length - 1]; - // console.log(`📅 API returned data from ${firstDatapoint[0]} to ${lastDatapoint[0]}`); - // console.log(`📊 Total datapoints in first item: ${firstItem.datapoints.length}`); - } - } + console.log(`✅ Request completed in ${endTime - startTime}ms`); const pageData = this.parseApplicationData(response.data, dataType) as T[]; - - // Log date range of parsed data - if (pageData.length > 0) { - const dates = pageData.map(item => item.date).filter(Boolean).sort(); - if (dates.length > 0) { - // console.log(`📅 Parsed data date range: ${dates[0]} to ${dates[dates.length - 1]}`); -} - } - allData = allData.concat(pageData); // Check pagination @@ -644,15 +580,9 @@ class AcquiaApiServiceFixed { if (pageInfo) { totalPages = pageInfo.totalPages || pageInfo.total_pages || 1; hasMorePages = currentPage < totalPages; - // console.log(`📄 Pagination: page ${currentPage} of ${totalPages}`); } else { const links = response.data._links; hasMorePages = !!(links && links.next); - if (links?.next) { - // console.log(`📄 Found next link: ${links.next.href}`); - } else { - // console.log(`📄 No more pages found`); - } } currentPage++; @@ -662,41 +592,16 @@ class AcquiaApiServiceFixed { } if (hasMorePages) { - // console.log('⏱️ Waiting 500ms before next request...'); await new Promise(resolve => setTimeout(resolve, 500)); } } catch (error) { console.error(`❌ Error fetching page ${currentPage}:`, error); - - if (error instanceof Error && error.message.includes('timeout')) { - throw new Error(`Request timed out after ${this.API_TIMEOUT / 1000} seconds. Try a smaller date range or check your network connection.`); - } - throw error; } } - this.reportProgress({ - step: `Completed! Collected ${allData.length} ${dataType} records.`, - currentPage: currentPage - 1, - totalPages, - itemsCollected: allData.length - }); - - // Store the fresh data in the cache - cache[cacheKey] = { data: allData, timestamp: Date.now() }; - - // console.log(`🎉 Successfully fetched ${allData.length} ${dataType} records from ${currentPage - 1} pages`); - - // Final summary of date range - if (allData.length > 0) { - const dates = allData.map(item => item.date).filter(Boolean).sort(); - if (dates.length > 0) { - // console.log(`📅 Final data covers: ${dates[0]} to ${dates[dates.length - 1]}`); - } - } - + console.log(`🎉 Successfully fetched ${allData.length} ${dataType} records from ${currentPage - 1} pages`); return allData; } diff --git a/lib/cache.ts b/lib/cache.ts new file mode 100644 index 0000000..2b2b129 --- /dev/null +++ b/lib/cache.ts @@ -0,0 +1,103 @@ +import fs from 'fs/promises'; +import path from 'path'; +import crypto from 'crypto'; + +const CACHE_DIR = path.join(process.cwd(), '.cache'); +const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours in milliseconds + +// Ensure cache directory exists +async function ensureCacheDir() { + try { + await fs.mkdir(CACHE_DIR, { recursive: true }); + } catch (error) { + // Directory might already exist, ignore error + } +} + +// Generate a cache key from request parameters +function generateCacheKey(params: Record): string { + const sortedParams = Object.keys(params) + .sort() + .reduce((obj: Record, key) => { + obj[key] = params[key]; + return obj; + }, {}); + + const paramString = JSON.stringify(sortedParams); + return crypto.createHash('md5').update(paramString).digest('hex'); +} + +// Get cache file path +function getCacheFilePath(key: string): string { + return path.join(CACHE_DIR, `${key}.json`); +} + +// Check if cache is valid (not expired) +async function isCacheValid(filePath: string): Promise { + try { + const stats = await fs.stat(filePath); + const now = Date.now(); + const cacheTime = stats.mtime.getTime(); + return (now - cacheTime) < CACHE_TTL; + } catch (error) { + return false; + } +} + +// Get cached data if valid +export async function getCachedData(cacheKey: string): Promise { + try { + await ensureCacheDir(); + const filePath = getCacheFilePath(cacheKey); + + if (await isCacheValid(filePath)) { + const data = await fs.readFile(filePath, 'utf-8'); + const parsed = JSON.parse(data); + console.log(`📦 Cache HIT for key: ${cacheKey}`); + return parsed.data as T; + } else { + console.log(`❌ Cache MISS/EXPIRED for key: ${cacheKey}`); + return null; + } + } catch (error) { + console.log(`❌ Cache ERROR for key: ${cacheKey}`, error); + return null; + } +} + +// Set cached data +export async function setCachedData(cacheKey: string, data: T): Promise { + try { + await ensureCacheDir(); + const filePath = getCacheFilePath(cacheKey); + + const cacheData = { + data, + timestamp: new Date().toISOString(), + ttl: CACHE_TTL + }; + + await fs.writeFile(filePath, JSON.stringify(cacheData, null, 2)); + console.log(`💾 Cache SET for key: ${cacheKey}`); + } catch (error) { + console.error(`❌ Cache SET ERROR for key: ${cacheKey}`, error); + } +} + +// Generate cache key for API requests +export function generateApiCacheKey(endpoint: string, params: Record): string { + return generateCacheKey({ endpoint, ...params }); +} + +// Clear all cache files (useful for debugging) +export async function clearAllCache(): Promise { + try { + const files = await fs.readdir(CACHE_DIR); + await Promise.all( + files.map(file => fs.unlink(path.join(CACHE_DIR, file))) + ); + console.log('🗑️ All cache cleared'); + } catch (error) { + console.error('❌ Error clearing cache:', error); + } +} \ No newline at end of file diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..51c7ae6 --- /dev/null +++ b/vercel.json @@ -0,0 +1,13 @@ +{ + "headers": [ + { + "source": "/api/acquia/(.*)", + "headers": [ + { + "key": "Cache-Control", + "value": "public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400" + } + ] + } + ] +} \ No newline at end of file From abeeae073737c49821cc0cc3327ae646e9d42b7f Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 12:18:57 -0700 Subject: [PATCH 02/44] Implement hybrid cache strategy with unstable_cache --- app/api/acquia/views/route.ts | 86 ++++++++++++++++----------------- app/api/acquia/visits/route.ts | 84 ++++++++++++++++---------------- app/api/cache/route.ts | 15 +++--- lib/cache-hybrid.ts | 87 ++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 94 deletions(-) create mode 100644 lib/cache-hybrid.ts diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 9dd74fb..237a602 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -1,8 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import AcquiaApiServiceFixed from '@/lib/acquia-api'; -import { getCachedData, setCachedData, generateApiCacheKey } from '@/lib/cache'; - -export const revalidate = 21600; // 6 hours cache at route level +import { getCachedApiData, generateApiCacheKey } from '@/lib/cache-hybrid'; export async function GET(request: NextRequest) { const searchParams = request.nextUrl.searchParams; @@ -20,58 +18,56 @@ export async function GET(request: NextRequest) { ); } - // Generate cache key + // Generate cache key with ALL parameters const cacheKey = generateApiCacheKey('views', { subscriptionUuid, - from: from || 'no-from', - to: to || 'no-to', - resolution: resolution || 'no-resolution' + from, + to, + resolution }); - console.log('🗝️ Views cache key:', cacheKey); - - // Check cache first - const cachedResult = await getCachedData(cacheKey); - if (cachedResult) { - console.log('📦 Returning cached views data'); - return NextResponse.json({ - ...cachedResult, - cached: true, - cacheKey - }); - } - try { - const apiService = new AcquiaApiServiceFixed({ - baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', - authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', - apiKey: process.env.ACQUIA_API_KEY!, - apiSecret: process.env.ACQUIA_API_SECRET!, - }); + // Use hybrid caching + const result = await getCachedApiData( + async () => { + // This function only runs on cache miss + const apiService = new AcquiaApiServiceFixed({ + baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', + authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', + apiKey: process.env.ACQUIA_API_KEY!, + apiSecret: process.env.ACQUIA_API_SECRET!, + }); - console.log('🔧 Fetching fresh views data from API...'); - const data = await apiService.getViewsDataByApplication( - subscriptionUuid, - from || undefined, - to || undefined, - resolution || undefined - ); + console.log('🔧 Fetching fresh views data from API...'); + const data = await apiService.getViewsDataByApplication( + subscriptionUuid, + from || undefined, + to || undefined, + resolution || undefined + ); - console.log('✅ Got fresh views data:', data.length); + console.log('✅ Got fresh views data:', data.length); - const result = { - data, - totalItems: data.length, - message: `Successfully fetched ${data.length} view records`, - cached: false, - timestamp: new Date().toISOString(), - cacheKey - }; + return { + data, + totalItems: data.length, + message: `Successfully fetched ${data.length} view records`, + cached: false, + timestamp: new Date().toISOString(), + cacheKey, + requestParams: { subscriptionUuid, from, to, resolution } // Add for debugging + }; + }, + cacheKey, + ['views', subscriptionUuid] // Cache tags + ); - // Cache the result - await setCachedData(cacheKey, result); + // Add cache status to response + const response = NextResponse.json({ + ...result, + cached: true, // This will be overridden by the actual cache status + }); - const response = NextResponse.json(result); response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); return response; diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index ae525c9..20141c1 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from 'next/server'; import AcquiaApiServiceFixed from '@/lib/acquia-api'; -import { getCachedData, setCachedData, generateApiCacheKey } from '@/lib/cache'; +import { getCachedApiData, generateApiCacheKey } from '@/lib/cache-hybrid'; export async function GET(request: NextRequest) { const searchParams = request.nextUrl.searchParams; @@ -18,58 +18,56 @@ export async function GET(request: NextRequest) { ); } - // Generate cache key + // Generate cache key with ALL parameters const cacheKey = generateApiCacheKey('visits', { subscriptionUuid, - from: from || 'no-from', - to: to || 'no-to', - resolution: resolution || 'no-resolution' + from, + to, + resolution }); - console.log('🗝️ Visits cache key:', cacheKey); - - // Check cache first - const cachedResult = await getCachedData(cacheKey); - if (cachedResult) { - console.log('📦 Returning cached visits data'); - return NextResponse.json({ - ...cachedResult, - cached: true, - cacheKey - }); - } - try { - const apiService = new AcquiaApiServiceFixed({ - baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', - authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', - apiKey: process.env.ACQUIA_API_KEY!, - apiSecret: process.env.ACQUIA_API_SECRET!, - }); + // Use hybrid caching + const result = await getCachedApiData( + async () => { + // This function only runs on cache miss + const apiService = new AcquiaApiServiceFixed({ + baseUrl: process.env.ACQUIA_API_BASE_URL || 'https://cloud.acquia.com/api', + authUrl: process.env.ACQUIA_AUTH_BASE_URL || 'https://accounts.acquia.com/api', + apiKey: process.env.ACQUIA_API_KEY!, + apiSecret: process.env.ACQUIA_API_SECRET!, + }); - console.log('🔧 Fetching fresh visits data from API...'); - const data = await apiService.getVisitsDataByApplication( - subscriptionUuid, - from || undefined, - to || undefined, - resolution || undefined - ); + console.log('🔧 Fetching fresh visits data from API...'); + const data = await apiService.getVisitsDataByApplication( + subscriptionUuid, + from || undefined, + to || undefined, + resolution || undefined + ); - console.log('✅ Got fresh visits data:', data.length); + console.log('✅ Got fresh visits data:', data.length); - const result = { - data, - totalItems: data.length, - message: `Successfully fetched ${data.length} visit records across all pages`, - cached: false, - timestamp: new Date().toISOString(), - cacheKey - }; + return { + data, + totalItems: data.length, + message: `Successfully fetched ${data.length} visit records across all pages`, + cached: false, + timestamp: new Date().toISOString(), + cacheKey, + requestParams: { subscriptionUuid, from, to, resolution } // Add for debugging + }; + }, + cacheKey, + ['visits', subscriptionUuid] // Cache tags + ); - // Cache the result - await setCachedData(cacheKey, result); + // Add cache status to response + const response = NextResponse.json({ + ...result, + cached: true, // This will be overridden by the actual cache status + }); - const response = NextResponse.json(result); response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); return response; diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index f99358f..003d3f3 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,27 +1,30 @@ import { NextRequest, NextResponse } from 'next/server'; -import { clearAllCache } from '@/lib/cache'; +import { invalidateCache } from '@/lib/cache-hybrid'; export async function DELETE(request: NextRequest) { try { - await clearAllCache(); + await invalidateCache(['acquia-api', 'views', 'visits']); return NextResponse.json({ - message: 'Cache cleared successfully', + message: 'Cache invalidated successfully', timestamp: new Date().toISOString() }); } catch (error) { - console.error('Cache clear error:', error); + console.error('Cache invalidation error:', error); return NextResponse.json( - { error: 'Failed to clear cache' }, + { error: 'Failed to invalidate cache' }, { status: 500 } ); } } export async function GET(request: NextRequest) { + const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; + return NextResponse.json({ message: 'Cache management API', + environment: isLocal ? 'Local (file cache)' : 'Production (unstable_cache)', endpoints: { - 'DELETE /api/cache': 'Clear all cached data' + 'DELETE /api/cache': 'Clear/invalidate all cached data' } }); } \ No newline at end of file diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts new file mode 100644 index 0000000..e9e1159 --- /dev/null +++ b/lib/cache-hybrid.ts @@ -0,0 +1,87 @@ +import { unstable_cache } from 'next/cache'; + +const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; + +// Hybrid caching: file cache for local development, unstable_cache for Vercel +export async function getCachedApiData( + apiCall: () => Promise, + cacheKey: string, + tags: string[] = [] +): Promise { + if (isLocal) { + console.log(`🏠 Using file cache for local development: ${cacheKey}`); + // Use your existing file cache for local development + const { getCachedData, setCachedData } = await import('./cache'); + const cached = await getCachedData(cacheKey); + if (cached) return cached; + + console.log(`🔥 File cache MISS - executing API call: ${cacheKey}`); + const result = await apiCall(); + await setCachedData(cacheKey, result); + return result; + } else { + console.log(`☁️ Using unstable_cache for Vercel: ${cacheKey}`); + // Use unstable_cache for Vercel + const cachedCall = unstable_cache( + async () => { + console.log(`🔥 unstable_cache MISS - executing API call: ${cacheKey}`); + return await apiCall(); + }, + [cacheKey], // Cache key array + { + revalidate: 6 * 60 * 60, // 6 hours in seconds + tags: ['acquia-api', ...tags] + } + ); + + return cachedCall(); + } +} + +// Generate cache key - FIXED to be more precise +export function generateApiCacheKey(endpoint: string, params: Record): string { + // Sort and normalize all parameters + const sortedParams = Object.keys(params) + .sort() + .reduce((obj: Record, key) => { + // Ensure all values are strings and handle null/undefined + obj[key] = params[key] ?? 'null'; + return obj; + }, {}); + + // Create a more detailed cache key + const keyComponents = [ + endpoint, + sortedParams.subscriptionUuid || 'no-sub', + sortedParams.from || 'no-from', + sortedParams.to || 'no-to', + sortedParams.resolution || 'no-res' + ]; + + // Join with a delimiter and create hash for consistent length + const keyString = keyComponents.join('|'); + console.log(`🗝️ Cache key components: ${keyString}`); + + // Use a shorter, more readable cache key + const crypto = require('crypto'); + const hash = crypto.createHash('md5').update(keyString).digest('hex').substring(0, 16); + const readableKey = `${endpoint}_${hash}`; + + console.log(`🔑 Final cache key: ${readableKey}`); + return readableKey; +} + +// Manual cache invalidation for Vercel (optional) +export async function invalidateCache(tags: string[] = ['acquia-api']) { + if (!isLocal) { + const { revalidateTag } = await import('next/cache'); + tags.forEach(tag => { + console.log(`🗑️ Invalidating cache tag: ${tag}`); + revalidateTag(tag); + }); + } else { + // Clear file cache in local development + const { clearAllCache } = await import('./cache'); + await clearAllCache(); + } +} \ No newline at end of file From 04e5804954121ac5b2dca24badba96c74577fe9b Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 12:30:24 -0700 Subject: [PATCH 03/44] Add clear cache button --- app/applications/[uuid]/page.tsx | 53 ++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 9e7e0c1..a50356d 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -41,6 +41,7 @@ export default function ApplicationDetailPage({ params }: any) { const [error, setError] = useState(null); const [dailyViews, setDailyViews] = useState([]); const [dailyVisits, setDailyVisits] = useState([]); + const [cacheClearing, setCacheClearing] = useState(false); // Fetch application name on mount or when subscriptionUuid changes useEffect(() => { @@ -159,11 +160,32 @@ export default function ApplicationDetailPage({ params }: any) { } }; + const clearCache = async () => { + setCacheClearing(true); + try { + const response = await fetch('/api/cache', { method: 'DELETE' }); + + if (response.ok) { + const result = await response.json(); + console.log('Server cache cleared:', result); + alert('Cache cleared successfully!'); + } else { + alert('Failed to clear cache'); + } + } catch (error) { + console.error('Cache clearing error:', error); + alert('Error clearing cache'); + } finally { + setCacheClearing(false); + } + }; + return (

Views and Visits Data for {appName ? appName : {uuid}}

+
-
+
+ + + {loading && (
@@ -235,11 +267,12 @@ export default function ApplicationDetailPage({ params }: any) { )}
-

+

(Note that it can take several minutes to fetch data from the Acquia API.)

+ {error && (
{error}
)} @@ -321,6 +354,7 @@ export default function ApplicationDetailPage({ params }: any) { loading, from, to, + cacheClearing, cacheInfo: { visitsUrl: `/api/acquia/visits?subscriptionUuid=${subscriptionUuid}&from=${from}&to=${to}&resolution=day`, viewsUrl: `/api/acquia/views?subscriptionUuid=${subscriptionUuid}&from=${from}&to=${to}&resolution=day`, @@ -342,16 +376,11 @@ export default function ApplicationDetailPage({ params }: any) { Clear Browser Cache
From 39ab22518f59e4aa1d1820ecdee2f0c421e8dedb Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 12:34:02 -0700 Subject: [PATCH 04/44] remove unused revalidate route --- app/api/revalidate/route.ts | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 app/api/revalidate/route.ts diff --git a/app/api/revalidate/route.ts b/app/api/revalidate/route.ts deleted file mode 100644 index 67cfcf9..0000000 --- a/app/api/revalidate/route.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { revalidateTag } from 'next/cache'; - -export async function POST(request: NextRequest) { - try { - const { tags } = await request.json(); - - if (!Array.isArray(tags)) { - return NextResponse.json({ error: 'tags must be an array' }, { status: 400 }); - } - - // Revalidate specified cache tags - for (const tag of tags) { - revalidateTag(tag); - } - - return NextResponse.json({ - message: `Revalidated tags: ${tags.join(', ')}`, - revalidated: true, - now: Date.now() - }); - } catch (error) { - console.error('Revalidation error:', error); - return NextResponse.json({ error: 'Failed to revalidate' }, { status: 500 }); - } -} \ No newline at end of file From f255fc4c428d00635888b2766d8a48bad7b63e58 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 12:43:36 -0700 Subject: [PATCH 05/44] Updated cache invalidation --- app/api/cache/route.ts | 46 +++++++++++++++++++++++++------- app/applications/[uuid]/page.tsx | 20 ++++++++++---- components/Dashboard.tsx | 28 +++++++++++-------- lib/cache-hybrid.ts | 14 +++++++--- 4 files changed, 79 insertions(+), 29 deletions(-) diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index 003d3f3..b132029 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,17 +1,44 @@ import { NextRequest, NextResponse } from 'next/server'; -import { invalidateCache } from '@/lib/cache-hybrid'; + +const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; export async function DELETE(request: NextRequest) { try { - await invalidateCache(['acquia-api', 'views', 'visits']); - return NextResponse.json({ - message: 'Cache invalidated successfully', - timestamp: new Date().toISOString() - }); + if (isLocal) { + // Local development - clear file cache + const { clearAllCache } = await import('@/lib/cache'); + await clearAllCache(); + return NextResponse.json({ + message: 'File cache cleared successfully', + environment: 'local', + timestamp: new Date().toISOString() + }); + } else { + // Vercel production - revalidate cache tags + const { revalidateTag } = await import('next/cache'); + + // Revalidate all the tags we use + const tags = ['acquia-api', 'views', 'visits']; + tags.forEach(tag => { + revalidateTag(tag); + console.log(`🗑️ Revalidated tag: ${tag}`); + }); + + return NextResponse.json({ + message: `Cache tags revalidated: ${tags.join(', ')}`, + environment: 'production', + timestamp: new Date().toISOString(), + revalidatedTags: tags + }); + } } catch (error) { - console.error('Cache invalidation error:', error); + console.error('Cache management error:', error); return NextResponse.json( - { error: 'Failed to invalidate cache' }, + { + error: 'Failed to clear cache', + details: error instanceof Error ? error.message : 'Unknown error', + environment: isLocal ? 'local' : 'production' + }, { status: 500 } ); } @@ -25,6 +52,7 @@ export async function GET(request: NextRequest) { environment: isLocal ? 'Local (file cache)' : 'Production (unstable_cache)', endpoints: { 'DELETE /api/cache': 'Clear/invalidate all cached data' - } + }, + availableTags: ['acquia-api', 'views', 'visits'] }); } \ No newline at end of file diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index a50356d..09a020f 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -163,18 +163,28 @@ export default function ApplicationDetailPage({ params }: any) { const clearCache = async () => { setCacheClearing(true); try { + console.log('🗑️ Attempting to clear cache...'); const response = await fetch('/api/cache', { method: 'DELETE' }); if (response.ok) { const result = await response.json(); - console.log('Server cache cleared:', result); - alert('Cache cleared successfully!'); + console.log('✅ Server cache cleared:', result); + + // Show more detailed success message + const environment = result.environment || 'unknown'; + const method = result.revalidatedTags ? + `Revalidated tags: ${result.revalidatedTags.join(', ')}` : + 'File cache cleared'; + + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}`); } else { - alert('Failed to clear cache'); + const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); + console.error('❌ Failed to clear cache:', errorData); + alert(`Failed to clear cache: ${errorData.error || 'Unknown error'}`); } } catch (error) { - console.error('Cache clearing error:', error); - alert('Error clearing cache'); + console.error('❌ Cache clearing error:', error); + alert(`Error clearing cache: ${error instanceof Error ? error.message : 'Network error'}`); } finally { setCacheClearing(false); } diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 2bf01c2..5c53c62 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -261,22 +261,28 @@ const Dashboard: React.FC = () => { const clearCache = async () => { setCacheClearing(true); try { - const response = await fetch('/api/revalidate', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - tags: ['applications', 'views', 'visits', subscriptionUuid] - }) - }); + console.log('🗑️ Attempting to clear cache...'); + const response = await fetch('/api/cache', { method: 'DELETE' }); if (response.ok) { - alert('Cache cleared successfully!'); + const result = await response.json(); + console.log('✅ Server cache cleared:', result); + + // Show more detailed success message + const environment = result.environment || 'unknown'; + const method = result.revalidatedTags ? + `Revalidated tags: ${result.revalidatedTags.join(', ')}` : + 'File cache cleared'; + + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}`); } else { - alert('Failed to clear cache'); + const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); + console.error('❌ Failed to clear cache:', errorData); + alert(`Failed to clear cache: ${errorData.error || 'Unknown error'}`); } } catch (error) { - console.error('Cache clearing error:', error); - alert('Error clearing cache'); + console.error('❌ Cache clearing error:', error); + alert(`Error clearing cache: ${error instanceof Error ? error.message : 'Network error'}`); } finally { setCacheClearing(false); } diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index e9e1159..2a5f350 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -21,6 +21,8 @@ export async function getCachedApiData( return result; } else { console.log(`☁️ Using unstable_cache for Vercel: ${cacheKey}`); + console.log(`🏷️ Cache tags: ${tags.join(', ')}`); + // Use unstable_cache for Vercel const cachedCall = unstable_cache( async () => { @@ -30,7 +32,7 @@ export async function getCachedApiData( [cacheKey], // Cache key array { revalidate: 6 * 60 * 60, // 6 hours in seconds - tags: ['acquia-api', ...tags] + tags: ['acquia-api', ...tags] // Always include base tag } ); @@ -71,17 +73,21 @@ export function generateApiCacheKey(endpoint: string, params: Record { + tagsToInvalidate.forEach(tag => { console.log(`🗑️ Invalidating cache tag: ${tag}`); revalidateTag(tag); }); + return { success: true, environment: 'production', tags: tagsToInvalidate }; } else { // Clear file cache in local development const { clearAllCache } = await import('./cache'); await clearAllCache(); + return { success: true, environment: 'local', method: 'file-clear' }; } } \ No newline at end of file From 6d2af42b112ce14671615b4309d2efbb9b39ec73 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 12:55:22 -0700 Subject: [PATCH 06/44] Updated cache invalidation with revalidatePath --- app/api/cache/route.ts | 48 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index b132029..f0d30d2 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { revalidatePath, revalidateTag } from 'next/cache'; const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; @@ -14,25 +15,55 @@ export async function DELETE(request: NextRequest) { timestamp: new Date().toISOString() }); } else { - // Vercel production - revalidate cache tags - const { revalidateTag } = await import('next/cache'); + // Vercel production - use both revalidatePath and revalidateTag + console.log('🗑️ Clearing Vercel cache...'); - // Revalidate all the tags we use + // Clear all cached API routes + const apiPaths = [ + '/api/acquia/views', + '/api/acquia/visits', + '/api/acquia/applications' + ]; + + apiPaths.forEach(path => { + try { + revalidatePath(path, 'page'); + console.log(`✅ Revalidated path: ${path}`); + } catch (error) { + console.error(`❌ Failed to revalidate path ${path}:`, error); + } + }); + + // Also try revalidating tags const tags = ['acquia-api', 'views', 'visits']; tags.forEach(tag => { - revalidateTag(tag); - console.log(`🗑️ Revalidated tag: ${tag}`); + try { + revalidateTag(tag); + console.log(`✅ Revalidated tag: ${tag}`); + } catch (error) { + console.error(`❌ Failed to revalidate tag ${tag}:`, error); + } }); + // Clear the entire data cache for the app + try { + revalidatePath('/', 'layout'); + console.log(`✅ Revalidated root layout`); + } catch (error) { + console.error(`❌ Failed to revalidate root:`, error); + } + return NextResponse.json({ - message: `Cache tags revalidated: ${tags.join(', ')}`, + message: 'Cache cleared successfully', environment: 'production', timestamp: new Date().toISOString(), - revalidatedTags: tags + revalidatedPaths: apiPaths, + revalidatedTags: tags, + note: 'Used both revalidatePath and revalidateTag for maximum coverage' }); } } catch (error) { - console.error('Cache management error:', error); + console.error('❌ Cache management error:', error); return NextResponse.json( { error: 'Failed to clear cache', @@ -53,6 +84,7 @@ export async function GET(request: NextRequest) { endpoints: { 'DELETE /api/cache': 'Clear/invalidate all cached data' }, + availablePaths: ['/api/acquia/views', '/api/acquia/visits', '/api/acquia/applications'], availableTags: ['acquia-api', 'views', 'visits'] }); } \ No newline at end of file From 5c80917e19386d91edb3f1fb9c8452f1ae79412d Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 13:11:16 -0700 Subject: [PATCH 07/44] Add cache buster --- lib/cache-hybrid.ts | 59 +++++++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 2a5f350..1a8d7c1 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -2,6 +2,14 @@ import { unstable_cache } from 'next/cache'; const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; +// Add a cache buster timestamp that gets updated when cache is cleared +let cacheBusterTimestamp = Date.now(); + +export function updateCacheBuster() { + cacheBusterTimestamp = Date.now(); + console.log('🔄 Cache buster updated:', cacheBusterTimestamp); +} + // Hybrid caching: file cache for local development, unstable_cache for Vercel export async function getCachedApiData( apiCall: () => Promise, @@ -10,7 +18,6 @@ export async function getCachedApiData( ): Promise { if (isLocal) { console.log(`🏠 Using file cache for local development: ${cacheKey}`); - // Use your existing file cache for local development const { getCachedData, setCachedData } = await import('./cache'); const cached = await getCachedData(cacheKey); if (cached) return cached; @@ -20,19 +27,20 @@ export async function getCachedApiData( await setCachedData(cacheKey, result); return result; } else { - console.log(`☁️ Using unstable_cache for Vercel: ${cacheKey}`); + // Include cache buster in the cache key for Vercel + const busteredCacheKey = `${cacheKey}_${cacheBusterTimestamp}`; + console.log(`☁️ Using unstable_cache for Vercel: ${busteredCacheKey}`); console.log(`🏷️ Cache tags: ${tags.join(', ')}`); - // Use unstable_cache for Vercel const cachedCall = unstable_cache( async () => { - console.log(`🔥 unstable_cache MISS - executing API call: ${cacheKey}`); + console.log(`🔥 unstable_cache MISS - executing API call: ${busteredCacheKey}`); return await apiCall(); }, - [cacheKey], // Cache key array + [busteredCacheKey], { - revalidate: 6 * 60 * 60, // 6 hours in seconds - tags: ['acquia-api', ...tags] // Always include base tag + revalidate: 6 * 60 * 60, + tags: ['acquia-api', ...tags] } ); @@ -75,17 +83,36 @@ export function generateApiCacheKey(endpoint: string, params: Record { - console.log(`🗑️ Invalidating cache tag: ${tag}`); - revalidateTag(tag); - }); - return { success: true, environment: 'production', tags: tagsToInvalidate }; + // Update the cache buster to force new cache keys + updateCacheBuster(); + + // Still try the revalidation APIs in case they help + try { + const { revalidateTag, revalidatePath } = await import('next/cache'); + const tagsToInvalidate = specificTags || ['acquia-api', 'views', 'visits']; + + tagsToInvalidate.forEach(tag => { + revalidateTag(tag); + console.log(`🗑️ Revalidated tag: ${tag}`); + }); + + const pathsToRevalidate = ['/api/acquia/views', '/api/acquia/visits', '/api/acquia/applications']; + pathsToRevalidate.forEach(path => { + revalidatePath(path); + console.log(`🗑️ Revalidated path: ${path}`); + }); + } catch (error) { + console.warn('Revalidation failed:', error); + } + + return { + success: true, + environment: 'production', + method: 'cache-buster', + cacheBusterTimestamp + }; } else { - // Clear file cache in local development const { clearAllCache } = await import('./cache'); await clearAllCache(); return { success: true, environment: 'local', method: 'file-clear' }; From aa0255bf1d9a1978fc282b268903d309785ccc0c Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 13:20:23 -0700 Subject: [PATCH 08/44] Update cache invalidation --- app/api/cache/route.ts | 67 ++++++++++-------------------------------- lib/cache-hybrid.ts | 57 ++++++++++++++++++++++------------- 2 files changed, 53 insertions(+), 71 deletions(-) diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index f0d30d2..a20fec8 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,72 +1,38 @@ import { NextRequest, NextResponse } from 'next/server'; -import { revalidatePath, revalidateTag } from 'next/cache'; const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; export async function DELETE(request: NextRequest) { try { if (isLocal) { - // Local development - clear file cache + // Local development - directly clear file cache + console.log('🏠 Clearing local file cache...'); const { clearAllCache } = await import('@/lib/cache'); await clearAllCache(); + return NextResponse.json({ message: 'File cache cleared successfully', environment: 'local', + method: 'file-clear', timestamp: new Date().toISOString() }); } else { - // Vercel production - use both revalidatePath and revalidateTag - console.log('🗑️ Clearing Vercel cache...'); - - // Clear all cached API routes - const apiPaths = [ - '/api/acquia/views', - '/api/acquia/visits', - '/api/acquia/applications' - ]; - - apiPaths.forEach(path => { - try { - revalidatePath(path, 'page'); - console.log(`✅ Revalidated path: ${path}`); - } catch (error) { - console.error(`❌ Failed to revalidate path ${path}:`, error); - } - }); - - // Also try revalidating tags - const tags = ['acquia-api', 'views', 'visits']; - tags.forEach(tag => { - try { - revalidateTag(tag); - console.log(`✅ Revalidated tag: ${tag}`); - } catch (error) { - console.error(`❌ Failed to revalidate tag ${tag}:`, error); - } - }); - - // Clear the entire data cache for the app - try { - revalidatePath('/', 'layout'); - console.log(`✅ Revalidated root layout`); - } catch (error) { - console.error(`❌ Failed to revalidate root:`, error); - } + // Production - use cache-buster approach + console.log('☁️ Invalidating Vercel cache using cache-buster...'); + const { invalidateCache } = await import('@/lib/cache-hybrid'); + const result = await invalidateCache(); return NextResponse.json({ - message: 'Cache cleared successfully', - environment: 'production', + message: 'Cache invalidation triggered successfully', timestamp: new Date().toISOString(), - revalidatedPaths: apiPaths, - revalidatedTags: tags, - note: 'Used both revalidatePath and revalidateTag for maximum coverage' + ...result }); } } catch (error) { console.error('❌ Cache management error:', error); return NextResponse.json( { - error: 'Failed to clear cache', + error: 'Failed to invalidate cache', details: error instanceof Error ? error.message : 'Unknown error', environment: isLocal ? 'local' : 'production' }, @@ -76,15 +42,14 @@ export async function DELETE(request: NextRequest) { } export async function GET(request: NextRequest) { - const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; - return NextResponse.json({ message: 'Cache management API', - environment: isLocal ? 'Local (file cache)' : 'Production (unstable_cache)', + environment: isLocal ? 'Local (file cache)' : 'Production (cache-buster)', endpoints: { - 'DELETE /api/cache': 'Clear/invalidate all cached data' + 'DELETE /api/cache': 'Invalidate all cached data' }, - availablePaths: ['/api/acquia/views', '/api/acquia/visits', '/api/acquia/applications'], - availableTags: ['acquia-api', 'views', 'visits'] + note: isLocal ? + 'Local development uses file-based cache clearing' : + 'Production uses cache-buster timestamps to force new cache keys' }); } \ No newline at end of file diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 1a8d7c1..058795f 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -2,15 +2,34 @@ import { unstable_cache } from 'next/cache'; const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; -// Add a cache buster timestamp that gets updated when cache is cleared -let cacheBusterTimestamp = Date.now(); +// Cache buster that gets updated when cache is cleared +// This will be stored in a way that persists across serverless function instances +let cacheBusterTimestamp: number | null = null; -export function updateCacheBuster() { - cacheBusterTimestamp = Date.now(); - console.log('🔄 Cache buster updated:', cacheBusterTimestamp); +// Get the current cache buster timestamp +async function getCacheBuster(): Promise { + if (isLocal) { + // For local development, just use a simple timestamp + return cacheBusterTimestamp || Date.now(); + } else { + // For Vercel, we'll use an environment variable or API call to get cache buster + // This is a simple approach - in production you might use a database or external store + if (cacheBusterTimestamp === null) { + cacheBusterTimestamp = Date.now(); + } + return cacheBusterTimestamp; + } +} + +// Update the cache buster to force cache invalidation +export async function updateCacheBuster(): Promise { + const newTimestamp = Date.now(); + cacheBusterTimestamp = newTimestamp; + console.log('🔄 Cache buster updated:', newTimestamp); + return newTimestamp; } -// Hybrid caching: file cache for local development, unstable_cache for Vercel +// Hybrid caching with cache busting export async function getCachedApiData( apiCall: () => Promise, cacheKey: string, @@ -28,18 +47,20 @@ export async function getCachedApiData( return result; } else { // Include cache buster in the cache key for Vercel - const busteredCacheKey = `${cacheKey}_${cacheBusterTimestamp}`; + const cacheBuster = await getCacheBuster(); + const busteredCacheKey = `${cacheKey}_${cacheBuster}`; console.log(`☁️ Using unstable_cache for Vercel: ${busteredCacheKey}`); console.log(`🏷️ Cache tags: ${tags.join(', ')}`); + console.log(`🔄 Cache buster: ${cacheBuster}`); const cachedCall = unstable_cache( async () => { console.log(`🔥 unstable_cache MISS - executing API call: ${busteredCacheKey}`); return await apiCall(); }, - [busteredCacheKey], + [busteredCacheKey], // This key will be unique per cache-buster timestamp { - revalidate: 6 * 60 * 60, + revalidate: 6 * 60 * 60, // 6 hours tags: ['acquia-api', ...tags] } ); @@ -48,18 +69,15 @@ export async function getCachedApiData( } } -// Generate cache key - FIXED to be more precise +// Generate cache key (same as before) export function generateApiCacheKey(endpoint: string, params: Record): string { - // Sort and normalize all parameters const sortedParams = Object.keys(params) .sort() .reduce((obj: Record, key) => { - // Ensure all values are strings and handle null/undefined obj[key] = params[key] ?? 'null'; return obj; }, {}); - // Create a more detailed cache key const keyComponents = [ endpoint, sortedParams.subscriptionUuid || 'no-sub', @@ -68,11 +86,9 @@ export function generateApiCacheKey(endpoint: string, params: Record Date: Fri, 10 Oct 2025 13:45:35 -0700 Subject: [PATCH 09/44] add debug environment route --- app/api/debug-env/route.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 app/api/debug-env/route.ts diff --git a/app/api/debug-env/route.ts b/app/api/debug-env/route.ts new file mode 100644 index 0000000..881b54d --- /dev/null +++ b/app/api/debug-env/route.ts @@ -0,0 +1,24 @@ +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(request: NextRequest) { + return NextResponse.json({ + allEnvVars: { + NODE_ENV: process.env.NODE_ENV, + VERCEL: process.env.VERCEL, + VERCEL_ENV: process.env.VERCEL_ENV, + VERCEL_URL: process.env.VERCEL_URL, + VERCEL_REGION: process.env.VERCEL_REGION, + }, + detectionResults: { + isVercel1: process.env.VERCEL === '1', + isVercelEnv: !!process.env.VERCEL_ENV, + isVercelUrl: !!process.env.VERCEL_URL, + nodeEnvDev: process.env.NODE_ENV === 'development', + nodeEnvProd: process.env.NODE_ENV === 'production', + }, + recommendedDetection: { + isVercel: process.env.VERCEL === '1' || !!process.env.VERCEL_ENV, + isLocal: process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV + } + }); +} \ No newline at end of file From 402c96f73f51519f687dac12025104bfd06de5aa Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 13:53:13 -0700 Subject: [PATCH 10/44] WIP environment detection. Cache status: Cache clearing works on local Cache persists across instances on Vercel Cache clearing does not work on Vercel --- app/api/cache/route.ts | 34 +++++++++++++++++++------- lib/cache-hybrid.ts | 54 ++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 34 deletions(-) diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index a20fec8..04d1565 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,9 +1,18 @@ import { NextRequest, NextResponse } from 'next/server'; -const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; +// Fixed environment detection based on actual Vercel env vars +const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; +const isVercel = !!process.env.VERCEL_ENV; export async function DELETE(request: NextRequest) { try { + console.log('🔍 Environment detection:', { + isLocal, + isVercel, + NODE_ENV: process.env.NODE_ENV, + VERCEL_ENV: process.env.VERCEL_ENV + }); + if (isLocal) { // Local development - directly clear file cache console.log('🏠 Clearing local file cache...'); @@ -17,7 +26,7 @@ export async function DELETE(request: NextRequest) { timestamp: new Date().toISOString() }); } else { - // Production - use cache-buster approach + // Vercel - use cache-buster approach from cache-hybrid console.log('☁️ Invalidating Vercel cache using cache-buster...'); const { invalidateCache } = await import('@/lib/cache-hybrid'); const result = await invalidateCache(); @@ -34,7 +43,13 @@ export async function DELETE(request: NextRequest) { { error: 'Failed to invalidate cache', details: error instanceof Error ? error.message : 'Unknown error', - environment: isLocal ? 'local' : 'production' + environment: isLocal ? 'local' : 'vercel', + debug: { + isLocal, + isVercel, + NODE_ENV: process.env.NODE_ENV, + VERCEL_ENV: process.env.VERCEL_ENV + } }, { status: 500 } ); @@ -44,12 +59,15 @@ export async function DELETE(request: NextRequest) { export async function GET(request: NextRequest) { return NextResponse.json({ message: 'Cache management API', - environment: isLocal ? 'Local (file cache)' : 'Production (cache-buster)', + environment: isLocal ? 'Local (file cache)' : 'Vercel (cache-buster)', endpoints: { - 'DELETE /api/cache': 'Invalidate all cached data' + 'DELETE /api/cache': 'Clear/invalidate all cached data' }, - note: isLocal ? - 'Local development uses file-based cache clearing' : - 'Production uses cache-buster timestamps to force new cache keys' + debug: { + isLocal, + isVercel, + NODE_ENV: process.env.NODE_ENV, + VERCEL_ENV: process.env.VERCEL_ENV + } }); } \ No newline at end of file diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 058795f..9fb6809 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,24 +1,25 @@ import { unstable_cache } from 'next/cache'; -const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL; +// Fixed environment detection based on actual Vercel env vars +const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; +const isVercel = !!process.env.VERCEL_ENV; // This is the reliable indicator + +console.log('🔍 Cache-hybrid environment detection:', { + isLocal, + isVercel, + NODE_ENV: process.env.NODE_ENV, + VERCEL_ENV: process.env.VERCEL_ENV +}); // Cache buster that gets updated when cache is cleared -// This will be stored in a way that persists across serverless function instances let cacheBusterTimestamp: number | null = null; // Get the current cache buster timestamp async function getCacheBuster(): Promise { - if (isLocal) { - // For local development, just use a simple timestamp - return cacheBusterTimestamp || Date.now(); - } else { - // For Vercel, we'll use an environment variable or API call to get cache buster - // This is a simple approach - in production you might use a database or external store - if (cacheBusterTimestamp === null) { - cacheBusterTimestamp = Date.now(); - } - return cacheBusterTimestamp; + if (cacheBusterTimestamp === null) { + cacheBusterTimestamp = Date.now(); } + return cacheBusterTimestamp; } // Update the cache buster to force cache invalidation @@ -29,7 +30,7 @@ export async function updateCacheBuster(): Promise { return newTimestamp; } -// Hybrid caching with cache busting +// Hybrid caching: file cache for local development, unstable_cache for Vercel export async function getCachedApiData( apiCall: () => Promise, cacheKey: string, @@ -37,6 +38,7 @@ export async function getCachedApiData( ): Promise { if (isLocal) { console.log(`🏠 Using file cache for local development: ${cacheKey}`); + // Use your existing file cache for local development const { getCachedData, setCachedData } = await import('./cache'); const cached = await getCachedData(cacheKey); if (cached) return cached; @@ -58,7 +60,7 @@ export async function getCachedApiData( console.log(`🔥 unstable_cache MISS - executing API call: ${busteredCacheKey}`); return await apiCall(); }, - [busteredCacheKey], // This key will be unique per cache-buster timestamp + [busteredCacheKey], { revalidate: 6 * 60 * 60, // 6 hours tags: ['acquia-api', ...tags] @@ -69,7 +71,7 @@ export async function getCachedApiData( } } -// Generate cache key (same as before) +// Generate cache key (same as working version) export function generateApiCacheKey(endpoint: string, params: Record): string { const sortedParams = Object.keys(params) .sort() @@ -97,10 +99,15 @@ export function generateApiCacheKey(endpoint: string, params: Record Date: Fri, 10 Oct 2025 14:07:25 -0700 Subject: [PATCH 11/44] WIP: environment detection --- lib/cache-hybrid.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 9fb6809..ef294f3 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,8 +1,8 @@ import { unstable_cache } from 'next/cache'; -// Fixed environment detection based on actual Vercel env vars +// FIXED: Use the same environment detection as the API route const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; -const isVercel = !!process.env.VERCEL_ENV; // This is the reliable indicator +const isVercel = !!process.env.VERCEL_ENV; console.log('🔍 Cache-hybrid environment detection:', { isLocal, @@ -36,9 +36,10 @@ export async function getCachedApiData( cacheKey: string, tags: string[] = [] ): Promise { + console.log(`🔍 getCachedApiData environment check: isLocal=${isLocal}, isVercel=${isVercel}`); + if (isLocal) { console.log(`🏠 Using file cache for local development: ${cacheKey}`); - // Use your existing file cache for local development const { getCachedData, setCachedData } = await import('./cache'); const cached = await getCachedData(cacheKey); if (cached) return cached; @@ -99,15 +100,20 @@ export function generateApiCacheKey(endpoint: string, params: Record Date: Fri, 10 Oct 2025 14:18:47 -0700 Subject: [PATCH 12/44] Move environment detection to runtime --- lib/cache-hybrid.ts | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index ef294f3..9221d9a 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,16 +1,5 @@ import { unstable_cache } from 'next/cache'; -// FIXED: Use the same environment detection as the API route -const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; -const isVercel = !!process.env.VERCEL_ENV; - -console.log('🔍 Cache-hybrid environment detection:', { - isLocal, - isVercel, - NODE_ENV: process.env.NODE_ENV, - VERCEL_ENV: process.env.VERCEL_ENV -}); - // Cache buster that gets updated when cache is cleared let cacheBusterTimestamp: number | null = null; @@ -36,7 +25,9 @@ export async function getCachedApiData( cacheKey: string, tags: string[] = [] ): Promise { - console.log(`🔍 getCachedApiData environment check: isLocal=${isLocal}, isVercel=${isVercel}`); + // Check environment at runtime, not module load time + const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; + console.log(`🔍 getCachedApiData environment check: isLocal=${isLocal}, NODE_ENV=${process.env.NODE_ENV}, VERCEL_ENV=${process.env.VERCEL_ENV}`); if (isLocal) { console.log(`🏠 Using file cache for local development: ${cacheKey}`); @@ -100,9 +91,13 @@ export function generateApiCacheKey(endpoint: string, params: Record Date: Fri, 10 Oct 2025 14:33:38 -0700 Subject: [PATCH 13/44] fixup! FE language; browser cache invalidation --- app/api/acquia/views/route.ts | 7 +++++-- app/applications/[uuid]/page.tsx | 6 ++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 237a602..2d906c1 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -65,10 +65,13 @@ export async function GET(request: NextRequest) { // Add cache status to response const response = NextResponse.json({ ...result, - cached: true, // This will be overridden by the actual cache status + cached: true }); - response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); + // Add headers to prevent browser caching + response.headers.set('Cache-Control', 'no-cache, no-store, must-revalidate'); + response.headers.set('Pragma', 'no-cache'); + response.headers.set('Expires', '0'); return response; } catch (error) { diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 09a020f..75f475b 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -170,11 +170,9 @@ export default function ApplicationDetailPage({ params }: any) { const result = await response.json(); console.log('✅ Server cache cleared:', result); - // Show more detailed success message + // Fix the message parsing const environment = result.environment || 'unknown'; - const method = result.revalidatedTags ? - `Revalidated tags: ${result.revalidatedTags.join(', ')}` : - 'File cache cleared'; + const method = result.method || 'unknown'; // Use result.method, not result.revalidatedTags alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}`); } else { From b195d0d27229391a77c06e7e060ef831b9b6aea5 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 14:48:50 -0700 Subject: [PATCH 14/44] WIP: move caching from API routes to pages. Add debug message --- app/api/acquia/views/route.ts | 5 ----- app/applications/[uuid]/page.tsx | 19 +++++++++++++++---- components/Dashboard.tsx | 19 ++++++++++++++----- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 2d906c1..4126d51 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -68,11 +68,6 @@ export async function GET(request: NextRequest) { cached: true }); - // Add headers to prevent browser caching - response.headers.set('Cache-Control', 'no-cache, no-store, must-revalidate'); - response.headers.set('Pragma', 'no-cache'); - response.headers.set('Expires', '0'); - return response; } catch (error) { console.error('❌ Views API Error:', error); diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 75f475b..ad8c5d7 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -164,17 +164,28 @@ export default function ApplicationDetailPage({ params }: any) { setCacheClearing(true); try { console.log('🗑️ Attempting to clear cache...'); + + // Clear browser cache first + if ('caches' in window) { + const cacheNames = await caches.keys(); + await Promise.all( + cacheNames.map(cacheName => caches.delete(cacheName)) + ); + console.log('🗑️ Cleared browser caches:', cacheNames); + } + + // Clear server cache const response = await fetch('/api/cache', { method: 'DELETE' }); if (response.ok) { const result = await response.json(); console.log('✅ Server cache cleared:', result); - - // Fix the message parsing + console.log('🔍 Full response object:', JSON.stringify(result, null, 2)); // ADD THIS LINE + // Fix the parsing - check what field actually contains the method const environment = result.environment || 'unknown'; - const method = result.method || 'unknown'; // Use result.method, not result.revalidatedTags + const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared`); } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 5c53c62..49d2a8c 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -262,19 +262,28 @@ const Dashboard: React.FC = () => { setCacheClearing(true); try { console.log('🗑️ Attempting to clear cache...'); + + // Clear browser cache first + if ('caches' in window) { + const cacheNames = await caches.keys(); + await Promise.all( + cacheNames.map(cacheName => caches.delete(cacheName)) + ); + console.log('🗑️ Cleared browser caches:', cacheNames); + } + + // Clear server cache const response = await fetch('/api/cache', { method: 'DELETE' }); if (response.ok) { const result = await response.json(); console.log('✅ Server cache cleared:', result); + console.log('🔍 Full response object:', JSON.stringify(result, null, 2)); // ADD THIS LINE - // Show more detailed success message const environment = result.environment || 'unknown'; - const method = result.revalidatedTags ? - `Revalidated tags: ${result.revalidatedTags.join(', ')}` : - 'File cache cleared'; + const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared`); } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); From 99c252cd37ac9f077d8c58b1b3975e59cbe75332 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 10 Oct 2025 15:07:44 -0700 Subject: [PATCH 15/44] Add documentation --- app/applications/[uuid]/page.tsx | 1 - components/Dashboard.tsx | 1 - docs/caching.md | 225 +++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 docs/caching.md diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index ad8c5d7..9984453 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -180,7 +180,6 @@ export default function ApplicationDetailPage({ params }: any) { if (response.ok) { const result = await response.json(); console.log('✅ Server cache cleared:', result); - console.log('🔍 Full response object:', JSON.stringify(result, null, 2)); // ADD THIS LINE // Fix the parsing - check what field actually contains the method const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 49d2a8c..53a7c72 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -278,7 +278,6 @@ const Dashboard: React.FC = () => { if (response.ok) { const result = await response.json(); console.log('✅ Server cache cleared:', result); - console.log('🔍 Full response object:', JSON.stringify(result, null, 2)); // ADD THIS LINE const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; diff --git a/docs/caching.md b/docs/caching.md new file mode 100644 index 0000000..3a72c40 --- /dev/null +++ b/docs/caching.md @@ -0,0 +1,225 @@ +# Caching System Documentation + +This application implements a hybrid caching system that provides fast data retrieval while maintaining data freshness across different environments. + +## Overview + +The caching system uses different strategies based on the deployment environment: + +- **Local Development**: File-based cache stored in `.cache/` directory +- **Production (Vercel)**: Persistent cache using Next.js `unstable_cache` with cache-busting for invalidation + +## Architecture + +### Core Components + +1. **`lib/cache-hybrid.ts`** - Main caching logic with environment detection +2. **`lib/cache.ts`** - File-based cache implementation for local development +3. **`app/api/cache/route.ts`** - Cache management API endpoint +4. **Cache clearing UI** - "Clear Cache" buttons in Dashboard and application detail pages + +### Environment Detection + +The system automatically detects the environment using: + +```typescript +const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; +const isVercel = !!process.env.VERCEL_ENV; +``` + +## Local Development Caching + +### How It Works +- Uses file-based cache stored in `.cache/` directory +- Cache files are named using MD5 hashes of cache keys +- Data is stored as JSON files with timestamps + +### Cache Structure +``` +.cache/ +├── views_abc123.json +├── visits_def456.json +└── applications_ghi789.json +``` + +### Cache Clearing +- **Manual**: Click "Clear Cache" button (empties entire `.cache/` directory) +- **Automatic**: Cache files respect TTL and are regenerated as needed + +## Production (Vercel) Caching + +### How It Works +- Uses Next.js `unstable_cache` for persistence across serverless function instances +- Cache keys include a cache-buster timestamp for invalidation +- Data persists across deployments and function restarts + +### Cache Behavior +- **Cache Generation**: Shared across all Vercel instances +- **Cache Persistence**: Survives deployments and scaling events +- **Cache Invalidation**: Uses cache-busting technique (updates timestamp in cache keys) + +### Cache Key Format +``` +endpoint_hash_timestamp +// Example: views_a1b2c3d4_1728854400000 +``` + +## API Routes with Caching + +All Acquia API routes use the hybrid caching system: + +- **`/api/acquia/views`** - Views data with 6-hour cache +- **`/api/acquia/visits`** - Visits data with 6-hour cache +- **`/api/acquia/applications`** - Application data with 6-hour cache + +### Cache Key Generation + +Cache keys are generated from all request parameters: + +```typescript +const cacheKey = generateApiCacheKey('views', { + subscriptionUuid: 'abc-123', + from: '2025-01-01', + to: '2025-01-31', + resolution: 'day' +}); +// Result: views_a1b2c3d4 +``` + +## Manual Cache Management + +### Cache Clearing UI + +Both the Dashboard and application detail pages include "Clear Cache" buttons that: + +1. Clear browser caches using the Cache API +2. Call `/api/cache` DELETE endpoint to clear server cache +3. Display success/failure feedback with environment information + +### Cache Management API + +**`GET /api/cache`** - Get cache system information +```json +{ + "message": "Cache management API", + "environment": "Local (file cache)" | "Vercel (cache-buster)", + "endpoints": { + "DELETE /api/cache": "Clear/invalidate all cached data" + } +} +``` + +**`DELETE /api/cache`** - Clear all cached data +```json +{ + "message": "Cache cleared successfully", + "environment": "local" | "vercel", + "method": "file-clear" | "cache-buster", + "timestamp": "2025-10-10T21:52:35.743Z" +} +``` + +## Cache Behavior Characteristics + +### Local Development +- ✅ **Fast subsequent requests** (file system cache) +- ✅ **Complete cache clearing** (removes all files) +- ✅ **Immediate invalidation** (files deleted instantly) +- ⚠️ **Single instance only** (not shared across processes) + +### Production (Vercel) +- ✅ **Fast subsequent requests** (persistent cache layer) +- ✅ **Cross-instance persistence** (shared across all serverless functions) +- ✅ **Survives deployments** (cache persists across releases) +- ✅ **Cache invalidation** (cache-busting technique) +- ⚠️ **Instance-specific invalidation** (cache clearing doesn't propagate across all instances immediately) + +## Cache Duration + +All cached data has a **6-hour TTL (21,600 seconds)**: + +```typescript +{ + revalidate: 6 * 60 * 60, // 6 hours + tags: ['acquia-api', ...tags] +} +``` + +## Debugging and Monitoring + +### Debug Information + +In development mode, pages show debug information including: + +- Cache key components +- Request parameters +- Cache hit/miss status +- Environment detection results + +### Console Logging + +The caching system provides detailed console logging: + +``` +🏠 Using file cache for local development: views_abc123 +🔥 File cache MISS - executing API call: views_abc123 +☁️ Using unstable_cache for Vercel: views_abc123_1728854400000 +🗑️ Cache buster updated: 1728854500000 +``` + +### Log Prefixes +- 🏠 Local development operations +- ☁️ Vercel/production operations +- 🔥 Cache miss (fresh API call) +- 📦 Cache hit (served from cache) +- 🗑️ Cache clearing operations +- 🔍 Environment detection +- 🗝️ Cache key generation + +## Troubleshooting + +### Cache Not Working +1. Check environment detection in console logs +2. Verify cache key generation is consistent +3. Ensure API responses are cacheable (no errors) + +### Cache Not Clearing +1. Check console for cache clearing logs +2. Verify environment detection is correct +3. For Vercel: Check if cache-buster timestamp is updating + +### Performance Issues +1. Monitor cache hit/miss ratios in console +2. Check if cache keys are too specific (preventing hits) +3. Verify TTL settings are appropriate for your use case + +## Best Practices + +### For Developers + +1. **Always test locally first** - The file cache makes debugging easier +2. **Monitor console logs** - They provide detailed caching behavior information +3. **Use cache clearing** - When testing data changes or debugging cache issues +4. **Check environment variables** - Ensure proper detection in different environments + +### For Cache Key Design + +1. **Include all relevant parameters** - Any parameter that affects the result should be in the cache key +2. **Normalize parameters** - Sort and handle null/undefined values consistently +3. **Use readable prefixes** - Makes debugging easier (`views_`, `visits_`, etc.) + +### For Cache Duration + +1. **6 hours is appropriate** for most Acquia analytics data (data doesn't change frequently) +2. **Consider shorter TTL** for more dynamic data +3. **Manual clearing available** for immediate updates when needed + +## Future Improvements + +Potential enhancements for the caching system: + +1. **Cross-instance invalidation** - Use external storage (Redis, database) for cache-buster timestamps +2. **Selective cache clearing** - Clear specific cache keys instead of all data +3. **Cache warming** - Pre-populate cache for common queries +4. **Cache analytics** - Track hit/miss ratios and performance metrics +5. **Configurable TTL** - Allow different cache durations per endpoint \ No newline at end of file From 077b5758424bbb1e4c97a60f7743ad10c38feb99 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 15 Oct 2025 15:26:04 -0700 Subject: [PATCH 16/44] Update cache lifetime to 1 minute for troubleshooting --- lib/cache-hybrid.ts | 2 +- lib/cache.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 9221d9a..4ad7398 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -54,7 +54,7 @@ export async function getCachedApiData( }, [busteredCacheKey], { - revalidate: 6 * 60 * 60, // 6 hours + revalidate: 60, // 1 minute in seconds (was 21600 = 6 hours) tags: ['acquia-api', ...tags] } ); diff --git a/lib/cache.ts b/lib/cache.ts index 2b2b129..efa999a 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -3,7 +3,7 @@ import path from 'path'; import crypto from 'crypto'; const CACHE_DIR = path.join(process.cwd(), '.cache'); -const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours in milliseconds +const CACHE_TTL = 60 * 1000; // 1 minute in milliseconds (was 6 hours) // Ensure cache directory exists async function ensureCacheDir() { From 6ab816e8aa157d52d23eb321a9408305dfb8f419 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 15 Oct 2025 15:36:33 -0700 Subject: [PATCH 17/44] WIP: update cache lifetime to 2 minutes --- lib/cache-hybrid.ts | 2 +- lib/cache.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 4ad7398..2f70860 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -54,7 +54,7 @@ export async function getCachedApiData( }, [busteredCacheKey], { - revalidate: 60, // 1 minute in seconds (was 21600 = 6 hours) + revalidate: 120, // 2 minutes in seconds (was 21600 = 6 hours) tags: ['acquia-api', ...tags] } ); diff --git a/lib/cache.ts b/lib/cache.ts index efa999a..33bf8f9 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -3,7 +3,7 @@ import path from 'path'; import crypto from 'crypto'; const CACHE_DIR = path.join(process.cwd(), '.cache'); -const CACHE_TTL = 60 * 1000; // 1 minute in milliseconds (was 6 hours) +const CACHE_TTL = 60 * 2 * 1000; // 2 minutes in milliseconds (was 6 hours) // Ensure cache directory exists async function ensureCacheDir() { From 1b4e20a789dd900be20aadc3e4a087006ee136cb Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 24 Oct 2025 14:16:56 -0700 Subject: [PATCH 18/44] fixup! resolve error from merge conflict resolution --- app/applications/[uuid]/page.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 9bc8e81..bbde67a 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -203,8 +203,8 @@ export default function ApplicationDetailPage({ params }: any) {

- Views and Visits Data for {appName ? appName : {typedParams.uuid}} -

+ Views and Visits Data for {appName ? appName : {uuid}} +
From 819d5446cca9a2cbb208c6b6226fb02cbe157592 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 12 Nov 2025 14:57:34 -0800 Subject: [PATCH 19/44] caching: application-layer timestamp validation --- docs/cache-invalidation-fix.md | 282 +++++++++++++++++++++++++++++++++ docs/caching.md | 151 ++++++++++++++---- lib/cache-hybrid.ts | 69 ++++++-- 3 files changed, 460 insertions(+), 42 deletions(-) create mode 100644 docs/cache-invalidation-fix.md diff --git a/docs/cache-invalidation-fix.md b/docs/cache-invalidation-fix.md new file mode 100644 index 0000000..70dd6dc --- /dev/null +++ b/docs/cache-invalidation-fix.md @@ -0,0 +1,282 @@ +# Cache Invalidation Fix - Technical Summary + +## Problem Statement + +Testing on Vercel revealed that cached data was persisting far longer than the configured 2-minute TTL: + +``` +Time | Event +--------|------------------------------------------------------- +1:51 PM | Generated data from 10/01 to 10/21 +1:56 PM | All browsers still serving cached data (5 min - expected) +2:08 PM | All browsers still serving cached data (17 min - PROBLEM) +2:17 PM | Re-deployed application +2:18 PM | All browsers STILL serving cached data after deploy +3:38 PM | All browsers STILL serving cached data (nearly 2 hours!) +``` + +**Expected behavior**: Cache should expire after 2 minutes +**Actual behavior**: Cache persisted for 2+ hours, even across deployments + +## Root Cause Analysis + +The issue was caused by Next.js `unstable_cache` behavior on Vercel: + +1. **`revalidate` parameter ignored**: Despite setting `revalidate: 120` (2 minutes), Vercel's cache layer was not respecting this value +2. **Persistent cache across deployments**: Cache survived deployments because cache keys didn't change +3. **Cache-buster insufficient**: The cache-buster timestamp was stored in a module-level variable that reset with each serverless function cold start, making it unreliable across instances + +### Why Next.js `revalidate` Doesn't Work on Vercel + +From Next.js documentation and community reports: +- `unstable_cache` is still experimental and behavior varies by environment +- Vercel's CDN and edge caching layers have their own persistence logic +- `revalidate` is a "suggestion" not a guarantee - Vercel may serve stale content longer +- Edge caching prioritizes performance over strict TTL adherence + +## Solution: Three-Layer Cache Invalidation + +### 1. Deployment-Based Cache Versioning (Automatic) + +Cache keys now include Vercel's deployment ID, which changes with every deploy: + +```typescript +function getCacheVersion(): string { + const deploymentId = process.env.VERCEL_DEPLOYMENT_ID || + process.env.VERCEL_GIT_COMMIT_SHA?.substring(0, 8) || + 'local'; + return deploymentId; +} + +// Cache key: views_abc123_vdpl_def456_1728854400000 +// ^^^^^^^^^^^^^^ deployment ID +``` + +**Effect**: Every deployment automatically gets fresh cache keys, invalidating old cache + +### 2. Application-Layer Timestamp Validation (Automatic) + +Cached data now includes a timestamp that's validated on every request: + +```typescript +// Store with timestamp +const cachedData = { + data: result, + cachedAt: new Date().toISOString(), + cacheKey: versionedCacheKey +}; + +// Validate age on retrieval +const age = Date.now() - new Date(cachedResult.cachedAt).getTime(); +if (age >= CACHE_TTL_MS) { + // Refetch fresh data + await updateCacheBuster(); + return getCachedApiData(apiCall, cacheKey, tags); +} +``` + +**Effect**: Cache expires reliably after 2 minutes, regardless of Next.js cache behavior + +### 3. Manual Cache-Busting (User-Triggered) + +When users click "Clear Cache", updates the cache-buster timestamp: + +```typescript +export async function updateCacheBuster(): Promise { + const newTimestamp = Date.now(); + cacheBusterTimestamp = newTimestamp; + return newTimestamp; +} + +// New cache key: views_abc123_vdpl_def456_1728854500000 +// ^^^^^^^^^^^^^^ updated timestamp +``` + +**Effect**: Immediate cache invalidation for all subsequent requests + +### 4. Removed Unreliable `revalidate` Parameter + +```typescript +// BEFORE (didn't work on Vercel) +unstable_cache(apiCall, [cacheKey], { + revalidate: 120, // ❌ Ignored by Vercel + tags: ['acquia-api'] +}) + +// AFTER (application controls expiration) +unstable_cache(apiCall, [cacheKey], { + // No revalidate - we validate timestamps ourselves + tags: ['acquia-api'] +}) +``` + +## Code Changes + +### File: `lib/cache-hybrid.ts` + +**Added**: +- `CACHE_TTL_MS` constant (2 minutes in milliseconds) +- `getCacheVersion()` - Returns deployment ID for cache versioning +- `isCacheDataValid()` - Validates cached data age + +**Modified**: +- `getCachedApiData()`: + - Wraps cached data with `{ data, cachedAt, cacheKey }` + - Includes deployment version in cache keys + - Validates timestamp on every retrieval + - Recursively refetches if expired + - Removed `revalidate` parameter + +**Impact**: +- Cache now expires reliably after 2 minutes +- Automatic invalidation on deploy +- No breaking changes to API + +### File: `lib/cache.ts` + +**No changes required** - File-based cache for local development already working correctly + +### File: `docs/caching.md` + +**Updated sections**: +- Overview: Added deployment versioning and timestamp validation +- Cache behavior characteristics: Updated Vercel section +- Cache duration: Explained application-layer validation +- Console logging: Added new log examples +- Troubleshooting: Added timestamp validation checks +- Added new "Implementation Details" section + +## Testing Recommendations + +### Local Testing +```bash +npm run dev +# Open http://localhost:3000 +# Fetch data, wait 2+ minutes, refresh +# Should see new API call after 2 minutes +``` + +### Vercel Testing +After deploying: + +1. **Test automatic expiration**: + - Load page, note timestamp in console + - Wait 2 minutes + - Refresh page + - Check console for "⏰ Cache expired" and fresh API call + +2. **Test deployment invalidation**: + - Load page, note cache version in console + - Deploy new version + - Refresh page + - Cache version should change, forcing fresh data + +3. **Test manual clearing**: + - Load page + - Click "Clear Cache" button + - Refresh page + - Should see fresh API call immediately + +### Expected Console Logs + +**First request (cache miss)**: +``` +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 +📦 Cache version (deployment): dpl_def456 +🔄 Cache buster: 1728854400000 +🔥 unstable_cache MISS - executing API call +``` + +**Second request within 2 minutes (cache hit)**: +``` +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 +✅ Cache data valid, age: 45s +``` + +**Request after 2 minutes (expired)**: +``` +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 +⏰ Cache expired: age=150s, ttl=120s +🔄 Cache buster updated: 1728854500000 +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854500000 +🔥 unstable_cache MISS - executing API call +``` + +## Environment Variables Used + +- `VERCEL_DEPLOYMENT_ID` (automatic on Vercel) - Primary deployment identifier +- `VERCEL_GIT_COMMIT_SHA` (automatic on Vercel) - Fallback deployment identifier +- `NODE_ENV` (set to 'production' on Vercel) - Environment detection +- `VERCEL_ENV` (automatic on Vercel) - Vercel environment detection + +No new environment variables required! + +## Performance Impact + +### Positive +- ✅ Faster subsequent requests (cache still works) +- ✅ No stale data after 2 minutes (reliable expiration) +- ✅ No stale data after deployments (automatic invalidation) +- ✅ Better cache hit rates (timestamp validation happens after cache lookup) + +### Neutral +- ⚠️ Minimal overhead from timestamp validation (< 1ms per request) +- ⚠️ Slightly longer cache keys due to deployment ID + +### No Negative Impact +- ✅ Same number of API calls to Acquia +- ✅ No additional database or storage requirements +- ✅ No changes to frontend code required + +## Future Considerations + +### If 2-Minute TTL Needs Adjustment + +Edit `CACHE_TTL_MS` in `lib/cache-hybrid.ts`: + +```typescript +// Current: 2 minutes +const CACHE_TTL_MS = 2 * 60 * 1000; + +// For 5 minutes: +const CACHE_TTL_MS = 5 * 60 * 1000; + +// For 1 hour: +const CACHE_TTL_MS = 60 * 60 * 1000; +``` + +### If Per-Endpoint TTL Needed + +Could pass TTL as parameter to `getCachedApiData()`: + +```typescript +export async function getCachedApiData( + apiCall: () => Promise, + cacheKey: string, + tags: string[] = [], + ttlMs: number = CACHE_TTL_MS // Add TTL parameter +): Promise +``` + +### If External Cache Needed + +For better cross-instance coordination, could use: +- Redis for cache-buster timestamp +- Vercel KV for shared state +- Database table for cache metadata + +Currently not needed - application-layer validation works across all instances. + +## Summary + +**Problem**: Vercel cache persisted 2+ hours despite 2-minute configuration +**Root cause**: Next.js `revalidate` parameter not respected on Vercel +**Solution**: Three-layer approach (deployment versioning + timestamp validation + manual busting) +**Result**: Reliable 2-minute cache expiration + automatic deployment invalidation +**Impact**: No breaking changes, better cache behavior, comprehensive logging + +## References + +- Commit `6ab816e8aa157d52d23eb321a9408305dfb8f419` - Updated cache lifetime to 2 minutes +- Next.js `unstable_cache` docs: https://nextjs.org/docs/app/api-reference/functions/unstable_cache +- Vercel environment variables: https://vercel.com/docs/projects/environment-variables/system-environment-variables diff --git a/docs/caching.md b/docs/caching.md index 3a72c40..1492763 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -6,8 +6,11 @@ This application implements a hybrid caching system that provides fast data retr The caching system uses different strategies based on the deployment environment: -- **Local Development**: File-based cache stored in `.cache/` directory -- **Production (Vercel)**: Persistent cache using Next.js `unstable_cache` with cache-busting for invalidation +- **Local Development**: File-based cache stored in `.cache/` directory with 2-minute TTL +- **Production (Vercel)**: Persistent cache using Next.js `unstable_cache` with: + - **Application-layer timestamp validation** (2-minute TTL) + - **Deployment-based cache versioning** (auto-invalidates on new deployments) + - **Manual cache-busting** (via "Clear Cache" button) ## Architecture @@ -50,20 +53,32 @@ const isVercel = !!process.env.VERCEL_ENV; ### How It Works - Uses Next.js `unstable_cache` for persistence across serverless function instances -- Cache keys include a cache-buster timestamp for invalidation -- Data persists across deployments and function restarts +- **Three-layer cache invalidation strategy**: + 1. **Deployment versioning**: Cache keys include `VERCEL_DEPLOYMENT_ID` - automatically invalidates on deploy + 2. **Application-layer TTL**: Validates cached data timestamp (2 minutes) regardless of Next.js cache + 3. **Manual cache-busting**: Updates timestamp in cache keys when user clicks "Clear Cache" +- Cached data includes timestamp for application-layer validation +- **Removes `revalidate` parameter** - Next.js `revalidate` doesn't work reliably on Vercel ### Cache Behavior - **Cache Generation**: Shared across all Vercel instances -- **Cache Persistence**: Survives deployments and scaling events -- **Cache Invalidation**: Uses cache-busting technique (updates timestamp in cache keys) +- **Cache Persistence**: Data may persist in Next.js cache, but application validates age +- **Cache Invalidation**: + - Automatic: New deployments use different cache keys (`VERCEL_DEPLOYMENT_ID` changes) + - Time-based: Application checks `cachedAt` timestamp and refetches if > 2 minutes old + - Manual: "Clear Cache" button updates cache-buster timestamp ### Cache Key Format ``` -endpoint_hash_timestamp -// Example: views_a1b2c3d4_1728854400000 +endpoint_hash_vDEPLOYMENT_ID_timestamp +// Example: views_a1b2c3d4_vdpl_abc123_1728854400000 ``` +**Key Components**: +- `endpoint_hash`: Base cache key from request parameters +- `vDEPLOYMENT_ID`: Vercel deployment ID (changes with each deploy) +- `timestamp`: Cache-buster timestamp (updated on manual invalidation) + ## API Routes with Caching All Acquia API routes use the hybrid caching system: @@ -130,21 +145,39 @@ Both the Dashboard and application detail pages include "Clear Cache" buttons th ### Production (Vercel) - ✅ **Fast subsequent requests** (persistent cache layer) - ✅ **Cross-instance persistence** (shared across all serverless functions) -- ✅ **Survives deployments** (cache persists across releases) -- ✅ **Cache invalidation** (cache-busting technique) -- ⚠️ **Instance-specific invalidation** (cache clearing doesn't propagate across all instances immediately) +- ✅ **Automatic deployment invalidation** (new `VERCEL_DEPLOYMENT_ID` = new cache keys) +- ✅ **Consistent TTL behavior** (application validates timestamps) +- ✅ **Manual cache clearing** (cache-busting updates timestamp) +- ⚠️ **Old cache may persist in Next.js layer** (but app won't use it if expired) ## Cache Duration -All cached data has a **6-hour TTL (21,600 seconds)**: +All cached data has a **2-minute TTL**: ```typescript +// Validated at application layer, not relying on Next.js revalidate +const CACHE_TTL_MS = 2 * 60 * 1000; // 2 minutes in milliseconds + +// Cached data includes timestamp { - revalidate: 6 * 60 * 60, // 6 hours - tags: ['acquia-api', ...tags] + data: result, + cachedAt: new Date().toISOString(), + cacheKey: versionedCacheKey +} + +// Age validation on every request +if (!isCacheDataValid(cachedResult.cachedAt)) { + // Refetch if older than 2 minutes + await updateCacheBuster(); + return getCachedApiData(apiCall, cacheKey, tags); } ``` +**Why application-layer validation?** +- Next.js `revalidate` parameter doesn't work reliably on Vercel +- Testing showed cached data persisting for 2+ hours despite `revalidate: 120` +- Application-layer timestamp checks ensure consistent 2-minute TTL behavior + ## Debugging and Monitoring ### Debug Information @@ -163,7 +196,11 @@ The caching system provides detailed console logging: ``` 🏠 Using file cache for local development: views_abc123 🔥 File cache MISS - executing API call: views_abc123 -☁️ Using unstable_cache for Vercel: views_abc123_1728854400000 +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 +📦 Cache version (deployment): dpl_def456 +🔄 Cache buster: 1728854400000 +⏰ Cache expired: age=150s, ttl=120s +✅ Cache data valid, age: 45s 🗑️ Cache buster updated: 1728854500000 ``` @@ -178,20 +215,34 @@ The caching system provides detailed console logging: ## Troubleshooting +### Cache Not Expiring After 2 Minutes +1. **Check console logs for timestamp validation**: + - Look for "⏰ Cache expired" or "✅ Cache data valid, age: Xs" + - Verify age is being calculated correctly +2. **Verify deployment ID is changing**: + - Check "📦 Cache version (deployment)" in logs + - Should be different after each deployment +3. **Check if cache-buster is working**: + - Look for "🔄 Cache buster" in logs + - Should update when "Clear Cache" is clicked + ### Cache Not Working 1. Check environment detection in console logs 2. Verify cache key generation is consistent 3. Ensure API responses are cacheable (no errors) +4. Check that `cachedAt` timestamp is being added to cached data ### Cache Not Clearing -1. Check console for cache clearing logs +1. Check console for cache clearing logs ("🗑️ Cache buster updated") 2. Verify environment detection is correct -3. For Vercel: Check if cache-buster timestamp is updating +3. For Vercel: Verify cache-buster timestamp is updating in subsequent requests +4. Check browser console for "Clear Cache" button feedback ### Performance Issues -1. Monitor cache hit/miss ratios in console -2. Check if cache keys are too specific (preventing hits) -3. Verify TTL settings are appropriate for your use case +1. Monitor cache hit/miss ratios in console ("🔥 unstable_cache MISS") +2. Check cache age in logs ("✅ Cache data valid, age: Xs") +3. Verify timestamp validation is working correctly +4. Check if deployment ID is stable (shouldn't change unless deploying) ## Best Practices @@ -210,16 +261,50 @@ The caching system provides detailed console logging: ### For Cache Duration -1. **6 hours is appropriate** for most Acquia analytics data (data doesn't change frequently) -2. **Consider shorter TTL** for more dynamic data -3. **Manual clearing available** for immediate updates when needed - -## Future Improvements - -Potential enhancements for the caching system: - -1. **Cross-instance invalidation** - Use external storage (Redis, database) for cache-buster timestamps -2. **Selective cache clearing** - Clear specific cache keys instead of all data -3. **Cache warming** - Pre-populate cache for common queries -4. **Cache analytics** - Track hit/miss ratios and performance metrics -5. **Configurable TTL** - Allow different cache durations per endpoint \ No newline at end of file +1. **2 minutes is appropriate** for most Acquia analytics data while keeping it relatively fresh +2. **Application-layer validation ensures consistent behavior** across all environments +3. **Deployment-based versioning** automatically invalidates stale cache on deploy +4. **Manual clearing always available** for immediate updates when needed +5. **Adjust `CACHE_TTL_MS`** in `lib/cache-hybrid.ts` if different TTL is needed + +## Implementation Details + +### Why This Approach? + +**Problem**: Next.js `unstable_cache` with `revalidate` parameter doesn't work reliably on Vercel: +- Testing showed cached data persisting for 2+ hours despite `revalidate: 120` (2 minutes) +- Cache persisted across deployments and manual invalidation attempts +- `revalidateTag()` and `revalidatePath()` had no effect + +**Solution**: Three-layer cache invalidation: + +1. **Deployment Versioning** (automatic): + - Cache keys include `VERCEL_DEPLOYMENT_ID` or `VERCEL_GIT_COMMIT_SHA` + - Every deployment gets fresh cache keys automatically + - Zero configuration required + +2. **Application-Layer TTL** (automatic): + ```typescript + // Check timestamp on every request + if (!isCacheDataValid(cachedResult.cachedAt)) { + // Fetch fresh data if > 2 minutes old + await updateCacheBuster(); + return getCachedApiData(apiCall, cacheKey, tags); + } + ``` + - Validates `cachedAt` timestamp in application code + - Doesn't rely on Next.js cache behavior + - Consistent 2-minute TTL guaranteed + +3. **Manual Cache-Busting** (user-triggered): + - "Clear Cache" button updates `cacheBusterTimestamp` + - New requests use new cache keys + - Immediate invalidation for all subsequent requests + +### Advantages + +- ✅ **Predictable TTL**: Application controls expiration, not Next.js +- ✅ **Automatic deployment invalidation**: No stale cache after deploys +- ✅ **Manual control**: Users can force refresh when needed +- ✅ **Cross-instance consistency**: All instances respect timestamp validation +- ✅ **Debugging**: Clear console logs show cache version, age, and validation \ No newline at end of file diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 2f70860..83271be 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,8 +1,22 @@ import { unstable_cache } from 'next/cache'; +// Cache TTL in milliseconds (2 minutes) +const CACHE_TTL_MS = 2 * 60 * 1000; + // Cache buster that gets updated when cache is cleared +// This is stored in memory but we also use deployment ID to bust cache on deploy let cacheBusterTimestamp: number | null = null; +// Get cache version based on deployment ID (changes with each deploy) +// This ensures cache is automatically invalidated on new deployments +function getCacheVersion(): string { + // Use Vercel deployment ID if available, otherwise use a timestamp + const deploymentId = process.env.VERCEL_DEPLOYMENT_ID || + process.env.VERCEL_GIT_COMMIT_SHA?.substring(0, 8) || + 'local'; + return deploymentId; +} + // Get the current cache buster timestamp async function getCacheBuster(): Promise { if (cacheBusterTimestamp === null) { @@ -19,6 +33,20 @@ export async function updateCacheBuster(): Promise { return newTimestamp; } +// Check if cached data is still valid based on timestamp +function isCacheDataValid(cachedTimestamp: string): boolean { + const now = Date.now(); + const cacheTime = new Date(cachedTimestamp).getTime(); + const age = now - cacheTime; + const isValid = age < CACHE_TTL_MS; + + if (!isValid) { + console.log(`⏰ Cache expired: age=${Math.round(age/1000)}s, ttl=${CACHE_TTL_MS/1000}s`); + } + + return isValid; +} + // Hybrid caching: file cache for local development, unstable_cache for Vercel export async function getCachedApiData( apiCall: () => Promise, @@ -40,26 +68,49 @@ export async function getCachedApiData( await setCachedData(cacheKey, result); return result; } else { - // Include cache buster in the cache key for Vercel + // Include cache version (deployment ID) and cache buster in the cache key + const cacheVersion = getCacheVersion(); const cacheBuster = await getCacheBuster(); - const busteredCacheKey = `${cacheKey}_${cacheBuster}`; - console.log(`☁️ Using unstable_cache for Vercel: ${busteredCacheKey}`); + const versionedCacheKey = `${cacheKey}_v${cacheVersion}_${cacheBuster}`; + + console.log(`☁️ Using unstable_cache for Vercel: ${versionedCacheKey}`); console.log(`🏷️ Cache tags: ${tags.join(', ')}`); - console.log(`🔄 Cache buster: ${cacheBuster}`); + console.log(`� Cache version (deployment): ${cacheVersion}`); + console.log(`�🔄 Cache buster: ${cacheBuster}`); const cachedCall = unstable_cache( async () => { - console.log(`🔥 unstable_cache MISS - executing API call: ${busteredCacheKey}`); - return await apiCall(); + console.log(`🔥 unstable_cache MISS - executing API call: ${versionedCacheKey}`); + const result = await apiCall(); + + // Wrap the result with timestamp for validation + return { + data: result, + cachedAt: new Date().toISOString(), + cacheKey: versionedCacheKey + }; }, - [busteredCacheKey], + [versionedCacheKey], { - revalidate: 120, // 2 minutes in seconds (was 21600 = 6 hours) + // Don't use revalidate as it doesn't work reliably on Vercel + // Instead, we'll validate timestamps in application code tags: ['acquia-api', ...tags] } ); - return cachedCall(); + const cachedResult = await cachedCall(); + + // Validate cache age at application level + if (!isCacheDataValid(cachedResult.cachedAt)) { + console.log(`🔄 Cache data expired, fetching fresh data`); + // Bust the cache by updating the timestamp and recursively calling + await updateCacheBuster(); + // This will use a new cache key, forcing a fresh call + return getCachedApiData(apiCall, cacheKey, tags); + } + + console.log(`✅ Cache data valid, age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); + return cachedResult.data; } } From adb9400e8d36b9d4210ecd68dd949002b528f776 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 12 Nov 2025 15:25:59 -0800 Subject: [PATCH 20/44] Update browser caching settings --- app/api/acquia/views/route.ts | 4 +++ app/api/acquia/visits/route.ts | 4 ++- components/Dashboard.tsx | 4 +-- docs/cache-invalidation-fix.md | 60 +++++++++++++++++++++++++++++----- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 4126d51..5322c6c 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -68,6 +68,10 @@ export async function GET(request: NextRequest) { cached: true }); + // Set cache headers: max-age=120 (2 minutes) to match server-side TTL + // private prevents CDN caching, must-revalidate forces validation after expiry + response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); + return response; } catch (error) { console.error('❌ Views API Error:', error); diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index 20141c1..c5d63ac 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -68,7 +68,9 @@ export async function GET(request: NextRequest) { cached: true, // This will be overridden by the actual cache status }); - response.headers.set('Cache-Control', 'public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400'); + // Set cache headers: max-age=120 (2 minutes) to match server-side TTL + // s-maxage=0 prevents CDN caching, no-store for sensitive data + response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); return response; } catch (error) { diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 53a7c72..cfb7ead 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -117,10 +117,10 @@ const Dashboard: React.FC = () => { ...(dateTo && { to: dateTo }), }); - // Add cache-friendly headers to requests + // Disable browser caching - let server-side cache handle it const fetchOptions = { headers: { - 'Cache-Control': 'public, max-age=21600', + 'Cache-Control': 'no-cache', }, }; diff --git a/docs/cache-invalidation-fix.md b/docs/cache-invalidation-fix.md index 70dd6dc..0bad184 100644 --- a/docs/cache-invalidation-fix.md +++ b/docs/cache-invalidation-fix.md @@ -20,11 +20,12 @@ Time | Event ## Root Cause Analysis -The issue was caused by Next.js `unstable_cache` behavior on Vercel: +The issue was caused by **multiple layers of caching**, not just Next.js `unstable_cache`: 1. **`revalidate` parameter ignored**: Despite setting `revalidate: 120` (2 minutes), Vercel's cache layer was not respecting this value 2. **Persistent cache across deployments**: Cache survived deployments because cache keys didn't change 3. **Cache-buster insufficient**: The cache-buster timestamp was stored in a module-level variable that reset with each serverless function cold start, making it unreliable across instances +4. **Browser caching**: **Critical** - API routes were sending `Cache-Control: max-age=21600` (6 hours) headers, causing browsers to cache responses and never revalidate with the server ### Why Next.js `revalidate` Doesn't Work on Vercel @@ -34,7 +35,31 @@ From Next.js documentation and community reports: - `revalidate` is a "suggestion" not a guarantee - Vercel may serve stale content longer - Edge caching prioritizes performance over strict TTL adherence -## Solution: Three-Layer Cache Invalidation +### Why Browser Caching Was the Real Culprit + +Even with server-side cache working correctly, **browser HTTP caching** was preventing fresh data: +- API routes were sending: `Cache-Control: public, max-age=21600` (6 hours) +- Browsers cached responses and served them without contacting the server +- Page refresh didn't help - browser returned cached response immediately +- Server-side cache validation never ran because requests never reached the server + +## Solution: Three-Layer Cache Invalidation + Browser Cache Control + +### 0. Fix Browser Caching (Critical First Step) + +API routes now send proper cache headers that align with server-side TTL: + +```typescript +// In app/api/acquia/visits/route.ts and views/route.ts +response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); +``` + +**Key changes**: +- `private` - Prevents CDN/proxy caching +- `max-age=120` - Browser can cache for 2 minutes (matches server TTL) +- `must-revalidate` - Forces browser to check with server after expiry + +**Effect**: Browser respects 2-minute TTL and actually makes requests to the server ### 1. Deployment-Based Cache Versioning (Automatic) @@ -127,9 +152,24 @@ unstable_cache(apiCall, [cacheKey], { - Recursively refetches if expired - Removed `revalidate` parameter +### Files: `app/api/acquia/visits/route.ts` and `app/api/acquia/views/route.ts` + +**Modified**: +- Response headers: Changed from `max-age=21600` (6 hours) to `max-age=120` (2 minutes) +- Added `private` and `must-revalidate` directives +- Ensures browser cache aligns with server-side cache TTL + +### File: `components/Dashboard.tsx` + +**Modified**: +- Fetch headers: Changed from `Cache-Control: public, max-age=21600` to `Cache-Control: no-cache` +- Prevents client from requesting long-lived cache +- Lets server-side cache control behavior + **Impact**: -- Cache now expires reliably after 2 minutes +- Cache now expires reliably after 2 minutes **on both server and browser** - Automatic invalidation on deploy +- Browser actually makes requests to server after 2 minutes - No breaking changes to API ### File: `lib/cache.ts` @@ -270,12 +310,16 @@ Currently not needed - application-layer validation works across all instances. ## Summary **Problem**: Vercel cache persisted 2+ hours despite 2-minute configuration -**Root cause**: Next.js `revalidate` parameter not respected on Vercel -**Solution**: Three-layer approach (deployment versioning + timestamp validation + manual busting) -**Result**: Reliable 2-minute cache expiration + automatic deployment invalidation -**Impact**: No breaking changes, better cache behavior, comprehensive logging +**Root causes**: +1. Next.js `revalidate` parameter not respected on Vercel +2. **Browser HTTP caching with 6-hour `max-age` preventing server requests** + +**Solution**: +1. Three-layer server approach (deployment versioning + timestamp validation + manual busting) +2. **Browser cache headers aligned with server TTL (2 minutes)** -## References +**Result**: Reliable 2-minute cache expiration on both server and browser + automatic deployment invalidation +**Impact**: No breaking changes, better cache behavior, comprehensive logging, **actual cache expiration**## References - Commit `6ab816e8aa157d52d23eb321a9408305dfb8f419` - Updated cache lifetime to 2 minutes - Next.js `unstable_cache` docs: https://nextjs.org/docs/app/api-reference/functions/unstable_cache From 2a129c3378e114329dbad9125a4e94e00e5c5125 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 12 Nov 2025 15:48:25 -0800 Subject: [PATCH 21/44] Disable browser caching --- app/api/acquia/views/route.ts | 8 +++-- app/api/acquia/visits/route.ts | 8 +++-- components/Dashboard.tsx | 9 ++++-- docs/cache-invalidation-fix.md | 54 +++++++++++++++++++++++++--------- 4 files changed, 56 insertions(+), 23 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 5322c6c..1409b0b 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -68,9 +68,11 @@ export async function GET(request: NextRequest) { cached: true }); - // Set cache headers: max-age=120 (2 minutes) to match server-side TTL - // private prevents CDN caching, must-revalidate forces validation after expiry - response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); + // Disable browser caching completely - server handles all caching + // no-store prevents any caching, no-cache forces revalidation + response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); + response.headers.set('Pragma', 'no-cache'); + response.headers.set('Expires', '0'); return response; } catch (error) { diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index c5d63ac..2a48746 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -68,9 +68,11 @@ export async function GET(request: NextRequest) { cached: true, // This will be overridden by the actual cache status }); - // Set cache headers: max-age=120 (2 minutes) to match server-side TTL - // s-maxage=0 prevents CDN caching, no-store for sensitive data - response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); + // Disable browser caching completely - server handles all caching + // no-store prevents any caching, no-cache forces revalidation + response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); + response.headers.set('Pragma', 'no-cache'); + response.headers.set('Expires', '0'); return response; } catch (error) { diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index cfb7ead..bce7476 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -117,10 +117,13 @@ const Dashboard: React.FC = () => { ...(dateTo && { to: dateTo }), }); - // Disable browser caching - let server-side cache handle it - const fetchOptions = { + // Disable browser caching completely - let server-side cache handle it + // Use cache: 'no-store' to prevent fetch API from using cached responses + const fetchOptions: RequestInit = { + cache: 'no-store', headers: { - 'Cache-Control': 'no-cache', + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', }, }; diff --git a/docs/cache-invalidation-fix.md b/docs/cache-invalidation-fix.md index 0bad184..ca03b66 100644 --- a/docs/cache-invalidation-fix.md +++ b/docs/cache-invalidation-fix.md @@ -43,23 +43,49 @@ Even with server-side cache working correctly, **browser HTTP caching** was prev - Page refresh didn't help - browser returned cached response immediately - Server-side cache validation never ran because requests never reached the server -## Solution: Three-Layer Cache Invalidation + Browser Cache Control +**Critical Discovery from Testing**: +- Even after changing to `max-age=120` (2 minutes), browser caching still interfered +- Browsers with old cached responses (6-hour TTL) continued serving stale data +- Hard refresh was required to bypass existing cache +- **Solution**: Disable browser caching entirely - use `no-store` to prevent any browser caching -### 0. Fix Browser Caching (Critical First Step) +## Solution: Server-Side Cache Only (No Browser Caching) -API routes now send proper cache headers that align with server-side TTL: +### Key Insight: Disable Browser Caching Entirely + +After testing, we discovered that **any browser caching conflicts with server-side cache TTL**. The solution is to disable browser caching completely and rely solely on server-side caching. + +### 0. Disable Browser Caching (Critical) + +**API routes** send headers that prevent ALL browser caching: ```typescript // In app/api/acquia/visits/route.ts and views/route.ts -response.headers.set('Cache-Control', 'private, max-age=120, must-revalidate'); +response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); +response.headers.set('Pragma', 'no-cache'); +response.headers.set('Expires', '0'); +``` + +**Client fetch** requests with cache disabled: + +```typescript +// In components/Dashboard.tsx +const fetchOptions: RequestInit = { + cache: 'no-store', // Prevents fetch API from using cached responses + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + }, +}; ``` **Key changes**: -- `private` - Prevents CDN/proxy caching -- `max-age=120` - Browser can cache for 2 minutes (matches server TTL) -- `must-revalidate` - Forces browser to check with server after expiry +- `no-store` - Browser must not cache response at all +- `no-cache` - Browser must revalidate with server on every request +- `must-revalidate` - Browser cannot serve stale content +- `Pragma: no-cache` - HTTP/1.0 backwards compatibility -**Effect**: Browser respects 2-minute TTL and actually makes requests to the server +**Effect**: Every request goes to the server, where our 2-minute cache logic runs ### 1. Deployment-Based Cache Versioning (Automatic) @@ -155,16 +181,16 @@ unstable_cache(apiCall, [cacheKey], { ### Files: `app/api/acquia/visits/route.ts` and `app/api/acquia/views/route.ts` **Modified**: -- Response headers: Changed from `max-age=21600` (6 hours) to `max-age=120` (2 minutes) -- Added `private` and `must-revalidate` directives -- Ensures browser cache aligns with server-side cache TTL +- Response headers: `no-store, no-cache, must-revalidate, proxy-revalidate` +- Added `Pragma: no-cache` and `Expires: 0` for maximum compatibility +- **Completely disables browser caching** - all caching happens server-side ### File: `components/Dashboard.tsx` **Modified**: -- Fetch headers: Changed from `Cache-Control: public, max-age=21600` to `Cache-Control: no-cache` -- Prevents client from requesting long-lived cache -- Lets server-side cache control behavior +- Fetch options: Added `cache: 'no-store'` to RequestInit +- Request headers: `Cache-Control: no-cache, no-store, must-revalidate` +- **Forces browser to always contact server** - never uses cached responses **Impact**: - Cache now expires reliably after 2 minutes **on both server and browser** From 5431cdcb6d1bdc43f215d702081ef4d1afb26f00 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Wed, 12 Nov 2025 16:38:42 -0800 Subject: [PATCH 22/44] Add cache- busting query parameter to internal API call. Changed Cache Mode from 'no-store' to 'reload' --- app/api/acquia/views/route.ts | 1 + app/api/acquia/visits/route.ts | 1 + components/Dashboard.tsx | 12 +++++-- docs/cache-invalidation-fix.md | 58 ++++++++++++++++++++++++++++------ 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 1409b0b..3a0c5ae 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -8,6 +8,7 @@ export async function GET(request: NextRequest) { const from = searchParams.get('from'); const to = searchParams.get('to'); const resolution = searchParams.get('resolution'); + // Note: t (timestamp) parameter is ignored - used only to force browser to make network request console.log('🔍 Views API called with params:', { subscriptionUuid, from, to, resolution }); diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index 2a48746..38cd1c8 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -8,6 +8,7 @@ export async function GET(request: NextRequest) { const from = searchParams.get('from'); const to = searchParams.get('to'); const resolution = searchParams.get('resolution'); + // Note: t (timestamp) parameter is ignored - used only to force browser to make network request console.log('🔍 Visits API called with params:', { subscriptionUuid, from, to, resolution }); diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index bce7476..da8cfa7 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -39,6 +39,7 @@ const Dashboard: React.FC = () => { const [applicationMap, setApplicationMap] = useState>({}); const [activeTab, setActiveTab] = useState(TABS[0].key); const [cacheClearing, setCacheClearing] = useState(false); + // const [cacheBuster, setCacheBuster] = useState(Date.now()); // TODO: For manual cache clearing const fetchApplications = async () => { if (!subscriptionUuid) return; @@ -115,12 +116,14 @@ const Dashboard: React.FC = () => { subscriptionUuid, ...(dateFrom && { from: dateFrom }), ...(dateTo && { to: dateTo }), + // Force unique request to prevent any caching + t: Date.now().toString(), }); // Disable browser caching completely - let server-side cache handle it - // Use cache: 'no-store' to prevent fetch API from using cached responses + // Use cache: 'reload' to force network request const fetchOptions: RequestInit = { - cache: 'no-store', + cache: 'reload', // Forces request to go to network, bypassing cache headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', @@ -282,10 +285,13 @@ const Dashboard: React.FC = () => { const result = await response.json(); console.log('✅ Server cache cleared:', result); + // TODO: Add cache buster update here when implementing manual cache invalidation + // setCacheBuster(Date.now()); + const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); diff --git a/docs/cache-invalidation-fix.md b/docs/cache-invalidation-fix.md index ca03b66..eb0d3f5 100644 --- a/docs/cache-invalidation-fix.md +++ b/docs/cache-invalidation-fix.md @@ -47,13 +47,21 @@ Even with server-side cache working correctly, **browser HTTP caching** was prev - Even after changing to `max-age=120` (2 minutes), browser caching still interfered - Browsers with old cached responses (6-hour TTL) continued serving stale data - Hard refresh was required to bypass existing cache -- **Solution**: Disable browser caching entirely - use `no-store` to prevent any browser caching +- **Even `cache: 'no-store'` and `no-cache` headers didn't reliably prevent browser caching** +- **`cache: 'no-store'` doesn't force network requests** - browsers still return cached responses +- **Solution**: Use `cache: 'reload'` + timestamp query parameter that changes on every request to force unique URLs## Solution: Server-Side Cache Only (No Browser Caching) -## Solution: Server-Side Cache Only (No Browser Caching) +### Key Insight: Force Network Requests with Unique URLs -### Key Insight: Disable Browser Caching Entirely +After extensive testing, we discovered that **browser HTTP caching is extremely persistent** even with `no-store` directives. The only reliable solution is to **force unique URLs on every request** so the browser cannot use cached responses. -After testing, we discovered that **any browser caching conflicts with server-side cache TTL**. The solution is to disable browser caching completely and rely solely on server-side caching. +**The approach**: +1. **Disable browser caching** with response headers (`no-store`) +2. **Add timestamp to every request** (`t=Date.now()`) - creates unique URL each time +3. **Server ignores timestamp** when generating cache keys - preserves server-side caching +4. **Use `cache: 'reload'`** in fetch options - forces network request + +**Result**: Browser always makes network request (unique URL), but server-side cache still works (ignores timestamp in cache key). ### 0. Disable Browser Caching (Critical) @@ -66,12 +74,19 @@ response.headers.set('Pragma', 'no-cache'); response.headers.set('Expires', '0'); ``` -**Client fetch** requests with cache disabled: +**Client fetch** requests with unique URLs on every request: ```typescript // In components/Dashboard.tsx +const params = new URLSearchParams({ + subscriptionUuid, + from: dateFrom, + to: dateTo, + t: Date.now().toString(), // Timestamp - changes on EVERY request, forcing unique URL +}); + const fetchOptions: RequestInit = { - cache: 'no-store', // Prevents fetch API from using cached responses + cache: 'reload', // Forces network request, bypassing browser cache headers: { 'Cache-Control': 'no-cache, no-store, must-revalidate', 'Pragma': 'no-cache', @@ -79,13 +94,35 @@ const fetchOptions: RequestInit = { }; ``` +**API routes ignore the timestamp parameter**: + +```typescript +// In app/api/acquia/visits/route.ts and views/route.ts +const subscriptionUuid = searchParams.get('subscriptionUuid'); +const from = searchParams.get('from'); +const to = searchParams.get('to'); +const resolution = searchParams.get('resolution'); +// Note: t (timestamp) parameter is ignored - used only to force browser to make network request + +// Cache key only uses meaningful parameters +const cacheKey = generateApiCacheKey('visits', { + subscriptionUuid, + from, + to, + resolution + // t is NOT included - same cache key for same data request +}); +``` + **Key changes**: - `no-store` - Browser must not cache response at all - `no-cache` - Browser must revalidate with server on every request - `must-revalidate` - Browser cannot serve stale content - `Pragma: no-cache` - HTTP/1.0 backwards compatibility +- **`cache: 'reload'`** - Forces network request (stronger than `no-store`) +- **`t` query parameter** - Changes on EVERY request, creating unique URL each time -**Effect**: Every request goes to the server, where our 2-minute cache logic runs +**Effect**: Every request goes to the server (unique URL = no cached response possible). Server checks cache age and returns cached data if < 2 minutes old. ### 1. Deployment-Based Cache Versioning (Automatic) @@ -188,9 +225,12 @@ unstable_cache(apiCall, [cacheKey], { ### File: `components/Dashboard.tsx` **Modified**: -- Fetch options: Added `cache: 'no-store'` to RequestInit +- Fetch requests include `t` query parameter with `Date.now()` timestamp +- Creates unique URL on every request to prevent browser caching +- Fetch options: Changed from `cache: 'no-store'` to `cache: 'reload'` - Request headers: `Cache-Control: no-cache, no-store, must-revalidate` -- **Forces browser to always contact server** - never uses cached responses +- **Forces browser to always make network request** - unique URL bypasses any cached response +- **Server-side cache still works** - timestamp parameter ignored in cache key generation **Impact**: - Cache now expires reliably after 2 minutes **on both server and browser** From e62c0cd474a3b8efe2cb5a3b8abc4dd69efab0b1 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 10:23:22 -0800 Subject: [PATCH 23/44] WIP debug --- components/Dashboard.tsx | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index da8cfa7..3fec3a4 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -308,6 +308,11 @@ const Dashboard: React.FC = () => { return (
+ {/* Deployment verification */} +
+ 🚀 Build: 2025-11-15 10:17PST - Cache Busting v2 +
+
This dashboard shows your monthly usage for Acquia Cloud hosting.
@@ -524,8 +529,12 @@ const Dashboard: React.FC = () => {
)} -
+ {/* Last updated timestamp to verify deployment */} +
+

Last updated: 2025-11-15 10:17PST

+
+ ); }; From 140d2632913d335b40098cd75d2d2cdf9d618741 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 14:12:30 -0800 Subject: [PATCH 24/44] 5 minute cache lifetime. update docs --- components/Dashboard.tsx | 9 - docs/caching.md | 358 +++++++++++++++++++++++++++++++++++---- lib/cache-hybrid.ts | 4 +- 3 files changed, 329 insertions(+), 42 deletions(-) diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 3fec3a4..555fab4 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -308,11 +308,6 @@ const Dashboard: React.FC = () => { return (
- {/* Deployment verification */} -
- 🚀 Build: 2025-11-15 10:17PST - Cache Busting v2 -
-
This dashboard shows your monthly usage for Acquia Cloud hosting.
@@ -530,10 +525,6 @@ const Dashboard: React.FC = () => {
)} - {/* Last updated timestamp to verify deployment */} -
-

Last updated: 2025-11-15 10:17PST

-
); }; diff --git a/docs/caching.md b/docs/caching.md index 1492763..cbf0e7f 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -1,53 +1,349 @@ -# Caching System Documentation +# Caching Implementation Documentation -This application implements a hybrid caching system that provides fast data retrieval while maintaining data freshness across different environments. +This document describes the hybrid caching system implemented for the Acquia Analytics Dashboard. ## Overview -The caching system uses different strategies based on the deployment environment: +The caching system uses two different approaches based on the environment: -- **Local Development**: File-based cache stored in `.cache/` directory with 2-minute TTL -- **Production (Vercel)**: Persistent cache using Next.js `unstable_cache` with: - - **Application-layer timestamp validation** (2-minute TTL) - - **Deployment-based cache versioning** (auto-invalidates on new deployments) - - **Manual cache-busting** (via "Clear Cache" button) +- **Local Development**: File-based caching using JSON files +- **Vercel Production**: Next.js `unstable_cache` with application-layer timestamp validation -## Architecture +The system provides: +- **5-minute cache duration** for expensive API calls +- **Automatic cache invalidation** on application deployment +- **Manual cache clearing** via API endpoint +- **Browser cache prevention** to ensure server-side cache control -### Core Components +## Cache Architecture -1. **`lib/cache-hybrid.ts`** - Main caching logic with environment detection -2. **`lib/cache.ts`** - File-based cache implementation for local development -3. **`app/api/cache/route.ts`** - Cache management API endpoint -4. **Cache clearing UI** - "Clear Cache" buttons in Dashboard and application detail pages +### Server-Side Caching Strategy -### Environment Detection +The implementation uses a **three-layer cache invalidation** approach: -The system automatically detects the environment using: +1. **Deployment-based versioning**: Cache keys include deployment ID, automatically invalidating cache on new deployments +2. **Application-layer timestamp validation**: Cached data includes timestamps that are validated on every request +3. **Manual cache-busting**: User-triggered cache invalidation updates cache keys immediately + +### Browser Cache Prevention + +**Critical**: The system completely disables browser caching to ensure all cache control happens server-side: + +- API responses include: `Cache-Control: no-store, no-cache, must-revalidate` +- Client requests include unique timestamp parameter (`t=Date.now()`) on every request +- Fetch options use `cache: 'reload'` to force network requests +- Server ignores timestamp parameter for cache key generation + +**Result**: Browser always makes network request (unique URL), but server-side cache still works efficiently. + +## Implementation Files + +### 1. `lib/cache-hybrid.ts` + +Main caching interface with environment detection and timestamp validation. + +**Key Functions:** +- `getCachedApiData(apiCall, cacheKey, tags)` - Main caching wrapper with timestamp validation +- `generateApiCacheKey(endpoint, params)` - Consistent cache key generation +- `invalidateCache(tags?)` - Manual cache clearing +- `getCacheVersion()` - Returns deployment ID for cache versioning +- `isCacheDataValid()` - Validates cached data age + +### 2. `lib/cache.ts` + +File-based caching for local development (unchanged). + +### 3. API Routes + +All Acquia API routes include browser cache prevention headers: ```typescript -const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; -const isVercel = !!process.env.VERCEL_ENV; +response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate'); +response.headers.set('Pragma', 'no-cache'); +response.headers.set('Expires', '0'); ``` -## Local Development Caching +## Cache Behavior by Environment -### How It Works -- Uses file-based cache stored in `.cache/` directory -- Cache files are named using MD5 hashes of cache keys -- Data is stored as JSON files with timestamps +### Local Development + +- **Storage**: JSON files in `.cache/` directory (gitignored) +- **Duration**: 5 minutes +- **Invalidation**: File deletion or manual clear +- **Key advantage**: Persists across server restarts + +### Vercel Production + +- **Storage**: Next.js `unstable_cache` with deployment versioning +- **Duration**: 5 minutes (application-layer validation) +- **Cache keys**: Include deployment ID (`VERCEL_DEPLOYMENT_ID`) +- **Invalidation**: Automatic on deploy + timestamp validation + manual busting +- **Note**: `revalidate` parameter removed as it's unreliable on Vercel -### Cache Structure +## Cache Duration + +All cached data has a **5-minute lifespan**: + +```typescript +const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes ``` -.cache/ -├── views_abc123.json -├── visits_def456.json -└── applications_ghi789.json + +This duration was chosen because: +- Balances performance with data freshness expectations +- Reduces load on Acquia API while allowing reasonable update frequency +- Matches user expectations for analytics data refresh + +## Cache Validation Process + +### Data Storage Format + +Cached data is wrapped with metadata: + +```typescript +{ + data: actualApiResponse, + cachedAt: "2025-11-15T10:30:00.000Z", // Timestamp when cached + cacheKey: "visits_abc123_vdpl_def456_1731668400000" // Versioned cache key +} ``` -### Cache Clearing -- **Manual**: Click "Clear Cache" button (empties entire `.cache/` directory) -- **Automatic**: Cache files respect TTL and are regenerated as needed +### Validation on Every Request + +```typescript +const age = Date.now() - new Date(cachedResult.cachedAt).getTime(); +if (age >= CACHE_TTL_MS) { + // Cache expired - bust cache and refetch + await updateCacheBuster(); + return getCachedApiData(apiCall, cacheKey, tags); // Recursive call with new cache key +} +``` + +**Effect**: Cache reliably expires after 5 minutes regardless of Next.js behavior. + +## Cache Key Generation + +### Versioned Cache Keys + +Cache keys now include deployment version and cache-buster timestamp: + +``` +Format: {endpoint}_{hash}_v{deploymentId}_{cacheBuster} +Example: visits_a1b2c3d4_vdpl_abc123_1731668400000 +``` + +### Key Components + +```typescript +const keyComponents = [ + endpoint, // 'visits' or 'views' + sortedParams.subscriptionUuid, // Subscription identifier + sortedParams.from, // Date range start + sortedParams.to, // Date range end + sortedParams.resolution // Time resolution + // Note: 't' timestamp parameter is excluded +]; +``` + +**Key characteristics:** +- Deterministic: Same API parameters always generate same base key +- Versioned: Deployment changes automatically invalidate cache +- Cache-buster aware: Manual invalidation changes all keys immediately + +## Race Condition Behavior + +### Concurrent Requests to Same Data + +**Scenario**: Two browsers request the same data simultaneously before either completes. + +**Timeline Example**: +``` +10:00:00 Browser A: Starts API call for visits (11/1 to 11/10) +10:00:30 Browser B: Starts API call for visits (11/1 to 11/10) +10:00:50 Browser A: Completes, stores result in cache +10:01:20 Browser B: Completes, overwrites cache with its result +``` + +**Result**: Browser B's data "wins" and is stored in cache (last-write-wins). + +**Impact**: +- Both browsers make expensive API calls (no request deduplication) +- Cache contains data from whichever request finished last +- Since underlying data is identical, the "wrong" result doesn't affect users +- For this application's usage patterns, this is acceptable + +**Note**: Next.js `unstable_cache` doesn't prevent duplicate concurrent requests to the same cache key. + +## Manual Cache Invalidation + +### API Endpoint: `DELETE /api/cache` + +Clears all cached data immediately by updating the cache-buster timestamp: + +```bash +curl -X DELETE https://your-app.vercel.app/api/cache +``` + +**Response:** +```json +{ + "success": true, + "environment": "vercel", + "method": "cache-buster", + "cacheBusterTimestamp": 1731668500000 +} +``` + +### Cross-Browser Cache Clearing + +The "Clear Cache" button works across all browsers because: +- Updates server-side cache-buster timestamp +- All subsequent requests use new cache keys +- No browser-specific state involved + +## Console Logging + +The caching system provides detailed console logs: + +### Cache Hit (Valid): +``` +☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668400000 +✅ Cache data valid, age: 45s +``` + +### Cache Miss (Expired): +``` +☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668400000 +⏰ Cache expired: age=350s, ttl=300s +🔄 Cache buster updated: 1731668500000 +☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668500000 +🔥 unstable_cache MISS - executing API call +``` + +### Deployment Invalidation: +``` +📦 Cache version (deployment): dpl_new789 // Changed from dpl_old456 +🔥 unstable_cache MISS - executing API call // Automatic cache miss +``` + +## Browser Cache Prevention Details + +### Why Browser Caching Was Problematic + +Original issue: +- API routes were sending `Cache-Control: max-age=21600` (6 hours) +- Browsers cached responses and never contacted server +- Server-side cache validation never ran +- Even `no-store` headers didn't reliably prevent caching + +### Solution: Unique URLs + Strong Headers + +**Client-side** (Dashboard.tsx): +```typescript +const params = new URLSearchParams({ + subscriptionUuid, + from: dateFrom, + to: dateTo, + t: Date.now().toString() // Unique on every request +}); + +fetch(`/api/acquia/visits?${params}`, { + cache: 'reload', // Force network request + headers: { 'Cache-Control': 'no-cache' } +}); +``` + +**Server-side** (API routes): +```typescript +// Ignore 't' parameter for cache key +const cacheKey = generateApiCacheKey('visits', { + subscriptionUuid, from, to, resolution + // 't' is NOT included +}); + +// Prevent browser caching +response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate'); +``` + +**Result**: Every request has unique URL (browser can't cache), but server cache key is the same (cache still works). + +## Performance Characteristics + +### Cache Hit Performance +- **Local**: ~1-5ms (file read + timestamp validation) +- **Vercel**: ~1-10ms (in-memory access + timestamp validation) + +### Cache Miss Performance +- Same as underlying API call (~60-120 seconds) +- Cache storage is async and non-blocking +- Timestamp validation overhead: < 1ms + +## Troubleshooting + +### "Cache not expiring after 5 minutes" + +1. Check console for timestamp validation logs: + ``` + ⏰ Cache expired: age=350s, ttl=300s + 🔄 Cache buster updated: [timestamp] + ``` + +2. Verify browser is making network requests (check Network tab for unique `t=` parameter) + +3. Check deployment ID hasn't changed unexpectedly + +### "Stale data after deployment" + +1. Verify `VERCEL_DEPLOYMENT_ID` changed: + ``` + 📦 Cache version (deployment): [new-id] + ``` + +2. Check console for automatic cache miss on new deployment + +### "Cache clearing doesn't work" + +1. Verify API response indicates success: + ```json + {"success": true, "method": "cache-buster"} + ``` + +2. Check subsequent requests use new cache-buster timestamp + +3. Ensure browser isn't caching the `/api/cache` response itself + +## Configuration + +### Adjusting Cache Duration + +Edit `CACHE_TTL_MS` in `lib/cache-hybrid.ts`: + +```typescript +// Current: 5 minutes +const CACHE_TTL_MS = 5 * 60 * 1000; + +// For 2 minutes: +const CACHE_TTL_MS = 2 * 60 * 1000; + +// For 10 minutes: +const CACHE_TTL_MS = 10 * 60 * 1000; +``` + +## Environment Variables + +- `VERCEL_DEPLOYMENT_ID` (automatic) - Primary deployment identifier +- `VERCEL_GIT_COMMIT_SHA` (automatic) - Fallback deployment identifier +- `NODE_ENV` - Environment detection +- `VERCEL_ENV` - Vercel environment detection + +No additional configuration required. + +## Security Considerations + +1. **Cache directory**: `.cache/` is gitignored +2. **Cache keys**: MD5 hashing prevents directory traversal +3. **Browser cache prevention**: Eliminates client-side cache security concerns +4. **Manual clearing**: No authentication required (internal use) +5. **Deployment versioning**: Automatic cache invalidation on code changes ## Production (Vercel) Caching diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 83271be..11b52fb 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,7 +1,7 @@ import { unstable_cache } from 'next/cache'; -// Cache TTL in milliseconds (2 minutes) -const CACHE_TTL_MS = 2 * 60 * 1000; +// Cache TTL in milliseconds (5 minutes) +const CACHE_TTL_MS = 5 * 60 * 1000; // Cache buster that gets updated when cache is cleared // This is stored in memory but we also use deployment ID to bust cache on deploy From 4500ed9f467d9b86f5f853acc7f34a1387d23180 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 14:19:06 -0800 Subject: [PATCH 25/44] Caching: Update applications pages to use same logic as homepage --- app/applications/[uuid]/page.tsx | 43 ++++++- app/applications/page.tsx | 210 +++++++++++++++++++++++++------ 2 files changed, 211 insertions(+), 42 deletions(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index bbde67a..97e1758 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -50,7 +50,23 @@ export default function ApplicationDetailPage({ params }: any) { try { setLoadingStep('Fetching application info...'); - const res = await fetch(`/api/acquia/applications?subscriptionUuid=${subscriptionUuid}`); + + // Add cache-busting parameter to force fresh request + const params = new URLSearchParams({ + subscriptionUuid, + t: Date.now().toString() + }); + + const fetchOptions: RequestInit = { + cache: 'reload', // Forces request to go to network, bypassing cache + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + }, + }; + + console.log('📱 Fetching applications with cache-busting parameter'); + const res = await fetch(`/api/acquia/applications?${params}`, fetchOptions); if (!res.ok) { console.error('applications API responded with non-OK status', res.status); setAppName(''); @@ -85,13 +101,25 @@ export default function ApplicationDetailPage({ params }: any) { if (subscriptionUuid) paramsObj.subscriptionUuid = subscriptionUuid; if (from) paramsObj.from = from; if (to) paramsObj.to = to; + // Add cache-busting parameter + paramsObj.t = Date.now().toString(); const dailyQuery = new URLSearchParams({ ...paramsObj, resolution: 'day' }).toString(); + // Disable browser caching completely - let server-side cache handle it + const fetchOptions: RequestInit = { + cache: 'reload', // Forces request to go to network, bypassing cache + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + }, + }; + setLoadingStep('Fetching views and visits...'); + console.log('📊 Fetching analytics data with cache-busting parameter'); const [dailyViewsRes, dailyVisitsRes] = await Promise.all([ - fetch(`/api/acquia/views?${dailyQuery}`), - fetch(`/api/acquia/visits?${dailyQuery}`), + fetch(`/api/acquia/views?${dailyQuery}`, fetchOptions), + fetch(`/api/acquia/visits?${dailyQuery}`, fetchOptions), ]); const [dailyViewsRaw, dailyVisitsRaw]: [AcquiaApiResponse, AcquiaApiResponse] = await Promise.all([ @@ -180,11 +208,16 @@ export default function ApplicationDetailPage({ params }: any) { if (response.ok) { const result = await response.json(); console.log('✅ Server cache cleared:', result); - // Fix the parsing - check what field actually contains the method + const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); + + // Reload data with fresh cache-busting parameter + if (subscriptionUuid) { + await fetchAppDetail(); + } } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); diff --git a/app/applications/page.tsx b/app/applications/page.tsx index 266a319..192ac3d 100644 --- a/app/applications/page.tsx +++ b/app/applications/page.tsx @@ -1,16 +1,28 @@ -import React from 'react'; +'use client'; -const BASE_URL = process.env.VERCEL_URL - ? `https://${process.env.VERCEL_URL}` - : process.env.NEXT_PUBLIC_BASE_URL - ? process.env.NEXT_PUBLIC_BASE_URL - : 'http://localhost:3000'; +import React, { useState, useEffect } from 'react'; + +async function fetchData(cacheBuster?: string) { + const params = new URLSearchParams(); + if (cacheBuster) { + params.set('t', cacheBuster); + } + + const queryString = params.toString(); + const suffix = queryString ? `?${queryString}` : ''; + + const fetchOptions: RequestInit = { + cache: 'reload', // Forces request to go to network, bypassing cache + headers: { + 'Cache-Control': 'no-cache, no-store, must-revalidate', + 'Pragma': 'no-cache', + }, + }; -async function fetchData() { const [appsRes, viewsRes, visitsRes] = await Promise.all([ - fetch(`${BASE_URL}/api/acquia/applications`), - fetch(`${BASE_URL}/api/acquia/views`), - fetch(`${BASE_URL}/api/acquia/visits`), + fetch(`/api/acquia/applications${suffix}`, fetchOptions), + fetch(`/api/acquia/views${suffix}`, fetchOptions), + fetch(`/api/acquia/visits${suffix}`, fetchOptions), ]); const [apps, viewsRaw, visitsRaw] = await Promise.all([ appsRes.ok ? appsRes.json() : [], @@ -50,37 +62,161 @@ function getAppStats(apps: any[], views: { map: (arg0: (v: any) => any[]) => Ite })); } -export default async function ApplicationsPage() { - const { apps, views, visits } = await fetchData(); - const stats = getAppStats(apps, views, visits); +export default function ApplicationsPage() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + const [apps, setApps] = useState([]); + const [views, setViews] = useState([]); + const [visits, setVisits] = useState([]); + const [stats, setStats] = useState([]); + const [cacheClearing, setCacheClearing] = useState(false); + + const loadData = async () => { + setLoading(true); + setError(null); + + try { + console.log('📊 Fetching applications data with cache-busting parameter'); + const { apps, views, visits } = await fetchData(Date.now().toString()); + const calculatedStats = getAppStats(apps, views, visits); + + setApps(apps); + setViews(views); + setVisits(visits); + setStats(calculatedStats); + + console.log('✅ Applications data loaded successfully'); + } catch (err) { + console.error('❌ Failed to load applications data:', err); + const errorMessage = err instanceof Error ? err.message : 'An error occurred while fetching data'; + setError(errorMessage); + } finally { + setLoading(false); + } + }; + + const clearCache = async () => { + setCacheClearing(true); + try { + console.log('🗑️ Attempting to clear cache...'); + + // Clear browser cache first + if ('caches' in window) { + const cacheNames = await caches.keys(); + await Promise.all( + cacheNames.map(cacheName => caches.delete(cacheName)) + ); + console.log('🗑️ Cleared browser caches:', cacheNames); + } + + // Clear server cache + const response = await fetch('/api/cache', { method: 'DELETE' }); + + if (response.ok) { + const result = await response.json(); + console.log('✅ Server cache cleared:', result); + + const environment = result.environment || 'unknown'; + const method = result.method || 'unknown'; + + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); + + // Reload data with fresh cache-busting parameter + await loadData(); + } else { + const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); + console.error('❌ Failed to clear cache:', errorData); + alert(`Failed to clear cache: ${errorData.error || 'Unknown error'}`); + } + } catch (error) { + console.error('❌ Cache clearing error:', error); + alert(`Error clearing cache: ${error instanceof Error ? error.message : 'Network error'}`); + } finally { + setCacheClearing(false); + } + }; + + // Load data on component mount + useEffect(() => { + loadData(); + }, []); return (
-

Application Views & Visits

- - - - - - - - - - - - - {stats.map(app => ( - - - - - - - +
+

Application Views & Visits

+ +
+ + + +
+
+ + {loading && ( +
+
Loading applications data...
+
+ )} + + {error && ( +
+
Error: {error}
+ +
+ )} + + {!loading && !error && stats.length > 0 && ( +
ApplicationUUIDViews% of ViewsVisits% of Visits
{app.name}{app.uuid}{app.views.toLocaleString()}{app.viewsPct.toFixed(1)}%{app.visits.toLocaleString()}{app.visitsPct.toFixed(1)}%
+ + + + + + + + - ))} - -
ApplicationUUIDViews% of ViewsVisits% of Visits
+ + + {stats.map(app => ( + + {app.name} + {app.uuid} + {app.views.toLocaleString()} + {app.viewsPct.toFixed(1)}% + {app.visits.toLocaleString()} + {app.visitsPct.toFixed(1)}% + + ))} + + + )} + + {!loading && !error && stats.length === 0 && ( +
+
No application data found
+
+ )}
); } \ No newline at end of file From 2327a8adf606451abab6fcc16d94dfc7baca64d0 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 14:28:53 -0800 Subject: [PATCH 26/44] WIP debugging --- app/applications/[uuid]/page.tsx | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 97e1758..37228a1 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -101,10 +101,23 @@ export default function ApplicationDetailPage({ params }: any) { if (subscriptionUuid) paramsObj.subscriptionUuid = subscriptionUuid; if (from) paramsObj.from = from; if (to) paramsObj.to = to; - // Add cache-busting parameter - paramsObj.t = Date.now().toString(); - - const dailyQuery = new URLSearchParams({ ...paramsObj, resolution: 'day' }).toString(); + paramsObj.resolution = 'day'; + + // Add cache-busting parameter AFTER building the main params + const cacheBustingParam = Date.now().toString(); + + // Build query string with cache-busting parameter + const baseQuery = new URLSearchParams(paramsObj).toString(); + const dailyQuery = `${baseQuery}&t=${cacheBustingParam}`; + + console.log('📊 API request parameters:', { + subscriptionUuid, + from, + to, + resolution: 'day', + cacheBuster: cacheBustingParam, + fullQuery: dailyQuery + }); // Disable browser caching completely - let server-side cache handle it const fetchOptions: RequestInit = { From 6a0efbe00cf03a9f6045c68c9e3a966f7b0a12c4 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 14:37:09 -0800 Subject: [PATCH 27/44] Update clear cache button functionality. Remove debugging code --- app/applications/[uuid]/page.tsx | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index 37228a1..adf2fff 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -65,7 +65,6 @@ export default function ApplicationDetailPage({ params }: any) { }, }; - console.log('📱 Fetching applications with cache-busting parameter'); const res = await fetch(`/api/acquia/applications?${params}`, fetchOptions); if (!res.ok) { console.error('applications API responded with non-OK status', res.status); @@ -110,15 +109,6 @@ export default function ApplicationDetailPage({ params }: any) { const baseQuery = new URLSearchParams(paramsObj).toString(); const dailyQuery = `${baseQuery}&t=${cacheBustingParam}`; - console.log('📊 API request parameters:', { - subscriptionUuid, - from, - to, - resolution: 'day', - cacheBuster: cacheBustingParam, - fullQuery: dailyQuery - }); - // Disable browser caching completely - let server-side cache handle it const fetchOptions: RequestInit = { cache: 'reload', // Forces request to go to network, bypassing cache @@ -129,7 +119,6 @@ export default function ApplicationDetailPage({ params }: any) { }; setLoadingStep('Fetching views and visits...'); - console.log('📊 Fetching analytics data with cache-busting parameter'); const [dailyViewsRes, dailyVisitsRes] = await Promise.all([ fetch(`/api/acquia/views?${dailyQuery}`, fetchOptions), fetch(`/api/acquia/visits?${dailyQuery}`, fetchOptions), @@ -226,11 +215,6 @@ export default function ApplicationDetailPage({ params }: any) { const method = result.method || 'unknown'; alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); - - // Reload data with fresh cache-busting parameter - if (subscriptionUuid) { - await fetchAppDetail(); - } } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); @@ -242,9 +226,7 @@ export default function ApplicationDetailPage({ params }: any) { } finally { setCacheClearing(false); } - }; - - return ( + }; return (
From 37b12b052c23cbef32724866a3463beda97fb361 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 15:16:29 -0800 Subject: [PATCH 28/44] WIP: debugging --- lib/cache-hybrid.ts | 63 ++++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index 11b52fb..b8b382b 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -17,6 +17,9 @@ function getCacheVersion(): string { return deploymentId; } +// Generate a unique instance ID for debugging +const instanceId = Math.random().toString(36).substring(2, 8); + // Get the current cache buster timestamp async function getCacheBuster(): Promise { if (cacheBusterTimestamp === null) { @@ -68,27 +71,30 @@ export async function getCachedApiData( await setCachedData(cacheKey, result); return result; } else { - // Include cache version (deployment ID) and cache buster in the cache key + // Use only deployment ID for cache versioning (no per-instance cache buster) + // This ensures consistent cache keys across all serverless instances const cacheVersion = getCacheVersion(); - const cacheBuster = await getCacheBuster(); - const versionedCacheKey = `${cacheKey}_v${cacheVersion}_${cacheBuster}`; + const versionedCacheKey = `${cacheKey}_v${cacheVersion}`; - console.log(`☁️ Using unstable_cache for Vercel: ${versionedCacheKey}`); - console.log(`🏷️ Cache tags: ${tags.join(', ')}`); - console.log(`� Cache version (deployment): ${cacheVersion}`); - console.log(`�🔄 Cache buster: ${cacheBuster}`); + console.log(`☁️ Instance ${instanceId}: Using unstable_cache for Vercel: ${versionedCacheKey}`); + console.log(`🏷️ Instance ${instanceId}: Cache tags: ${tags.join(', ')}`); + console.log(`📦 Instance ${instanceId}: Cache version (deployment): ${cacheVersion}`); const cachedCall = unstable_cache( async () => { - console.log(`🔥 unstable_cache MISS - executing API call: ${versionedCacheKey}`); + console.log(`🔥 Instance ${instanceId}: unstable_cache MISS - executing API call: ${versionedCacheKey}`); const result = await apiCall(); // Wrap the result with timestamp for validation - return { + const cachedData = { data: result, cachedAt: new Date().toISOString(), - cacheKey: versionedCacheKey + cacheKey: versionedCacheKey, + instanceId: instanceId }; + + console.log(`💾 Instance ${instanceId}: Caching new data with timestamp: ${cachedData.cachedAt}`); + return cachedData; }, [versionedCacheKey], { @@ -100,16 +106,26 @@ export async function getCachedApiData( const cachedResult = await cachedCall(); + console.log(`📊 Instance ${instanceId}: Retrieved cached data from instance ${cachedResult.instanceId || 'unknown'}, cached at: ${cachedResult.cachedAt}`); + // Validate cache age at application level if (!isCacheDataValid(cachedResult.cachedAt)) { - console.log(`🔄 Cache data expired, fetching fresh data`); - // Bust the cache by updating the timestamp and recursively calling - await updateCacheBuster(); - // This will use a new cache key, forcing a fresh call - return getCachedApiData(apiCall, cacheKey, tags); + console.log(`🔄 Instance ${instanceId}: Cache data expired, fetching fresh data`); + console.log(`⏰ Instance ${instanceId}: Cache was created at: ${cachedResult.cachedAt}`); + console.log(`⏰ Instance ${instanceId}: Current time: ${new Date().toISOString()}`); + console.log(`⏰ Instance ${instanceId}: Cache age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); + + // For expired cache, we need to force a fresh API call + // Since we can't modify unstable_cache keys dynamically, we'll just call the API directly + console.log(`🆕 Instance ${instanceId}: Making fresh API call due to expired cache`); + const freshResult = await apiCall(); + + // Note: We can't easily update the existing cache entry, but that's okay + // The next request will get a fresh cache entry with the current timestamp + return freshResult; } - console.log(`✅ Cache data valid, age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); + console.log(`✅ Instance ${instanceId}: Cache data valid, age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); return cachedResult.data; } } @@ -158,11 +174,12 @@ export async function invalidateCache(specificTags?: string[]) { await clearAllCache(); return { success: true, environment: 'local', method: 'file-clear' }; } else { - // Vercel - update cache buster to force new cache keys - console.log('☁️ Running VERCEL cache invalidation (cache buster)'); - const newCacheBuster = await updateCacheBuster(); + // Vercel - with deployment-only cache keys, manual invalidation is limited + // Cache will be automatically invalidated on next deployment + console.log('☁️ Running VERCEL cache invalidation (deployment-based)'); + console.log('ℹ️ Note: Cache uses deployment ID only - will be cleared on next deploy'); - // Still try the revalidation APIs as a backup + // Still try the revalidation APIs as they may help in some cases try { const { revalidateTag, revalidatePath } = await import('next/cache'); const tagsToInvalidate = specificTags || ['acquia-api', 'views', 'visits']; @@ -178,14 +195,14 @@ export async function invalidateCache(specificTags?: string[]) { console.log(`🗑️ Revalidated path: ${path}`); }); } catch (error) { - console.warn('Revalidation APIs failed (expected with cache busting):', error); + console.warn('Revalidation APIs failed (expected with deployment-based caching):', error); } return { success: true, environment: 'vercel', - method: 'cache-buster', - cacheBusterTimestamp: newCacheBuster + method: 'deployment-based', + note: 'Cache will be cleared on next deployment' }; } } \ No newline at end of file From 9c738d601359a35010ce31e93aaceb136bfb1580 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 15:34:59 -0800 Subject: [PATCH 29/44] Doxn. Removed debugging code --- docs/caching.md | 217 ++++++++++++++++++++++++++++++-------------- lib/cache-hybrid.ts | 33 +++---- 2 files changed, 160 insertions(+), 90 deletions(-) diff --git a/docs/caching.md b/docs/caching.md index cbf0e7f..19e5680 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -19,11 +19,11 @@ The system provides: ### Server-Side Caching Strategy -The implementation uses a **three-layer cache invalidation** approach: +The implementation uses a **deployment-based cache invalidation** approach: -1. **Deployment-based versioning**: Cache keys include deployment ID, automatically invalidating cache on new deployments +1. **Deployment-only versioning**: Cache keys include deployment ID only, automatically invalidating cache on new deployments 2. **Application-layer timestamp validation**: Cached data includes timestamps that are validated on every request -3. **Manual cache-busting**: User-triggered cache invalidation updates cache keys immediately +3. **Manual cache clearing**: Limited to revalidation APIs (cache automatically clears on next deployment) ### Browser Cache Prevention @@ -112,23 +112,24 @@ Cached data is wrapped with metadata: ```typescript const age = Date.now() - new Date(cachedResult.cachedAt).getTime(); if (age >= CACHE_TTL_MS) { - // Cache expired - bust cache and refetch - await updateCacheBuster(); - return getCachedApiData(apiCall, cacheKey, tags); // Recursive call with new cache key + // Cache expired - make fresh API call directly + console.log('🆕 Making fresh API call due to expired cache'); + const freshResult = await apiCall(); + return freshResult; } ``` -**Effect**: Cache reliably expires after 5 minutes regardless of Next.js behavior. +**Effect**: Cache reliably expires after 5 minutes. Fresh data bypasses cache when expired. ## Cache Key Generation ### Versioned Cache Keys -Cache keys now include deployment version and cache-buster timestamp: +Cache keys now include deployment version only: ``` -Format: {endpoint}_{hash}_v{deploymentId}_{cacheBuster} -Example: visits_a1b2c3d4_vdpl_abc123_1731668400000 +Format: {endpoint}_{hash}_v{deploymentId} +Example: visits_a1b2c3d4_vdpl_abc123 ``` ### Key Components @@ -147,7 +148,7 @@ const keyComponents = [ **Key characteristics:** - Deterministic: Same API parameters always generate same base key - Versioned: Deployment changes automatically invalidate cache -- Cache-buster aware: Manual invalidation changes all keys immediately +- Stable within deployment: No per-request changes ## Race Condition Behavior @@ -177,7 +178,7 @@ const keyComponents = [ ### API Endpoint: `DELETE /api/cache` -Clears all cached data immediately by updating the cache-buster timestamp: +Attempts to clear cached data using Next.js revalidation APIs: ```bash curl -X DELETE https://your-app.vercel.app/api/cache @@ -188,17 +189,58 @@ curl -X DELETE https://your-app.vercel.app/api/cache { "success": true, "environment": "vercel", - "method": "cache-buster", - "cacheBusterTimestamp": 1731668500000 + "method": "deployment-based", + "note": "Cache will be cleared on next deployment" } ``` -### Cross-Browser Cache Clearing +**Note**: With deployment-only cache keys, manual cache clearing has limited effect. Cache is automatically cleared on the next deployment. -The "Clear Cache" button works across all browsers because: -- Updates server-side cache-buster timestamp -- All subsequent requests use new cache keys -- No browser-specific state involved +### Cache Clearing Behavior + +The "Clear Cache" button has limited effect in production because: +- Cache keys are based only on deployment ID (no per-request busting) +- Attempts to use Next.js revalidation APIs which have limited effectiveness +- Cache will be fully cleared on next deployment +- Browser cache is still cleared successfully + +## Cross-Instance Cache Sharing + +### Vercel Serverless Instance Behavior + +Testing revealed that `unstable_cache` **works correctly across different Vercel serverless instances**: + +**Evidence from production logs:** +``` +📊 Instance 1xk162: Retrieved cached data from instance 4czmjg, cached at: 2025-11-14T23:20:31.122Z +✅ Instance 1xk162: Cache data valid, age: 135s +``` + +**This proves:** +- ✅ **Instance `1xk162`** successfully retrieved cached data originally created by **Instance `4czmjg`** +- ✅ **Cross-instance cache sharing works perfectly** +- ✅ **Age calculations are accurate** across instances +- ✅ **Cache consistency** is maintained in Vercel's serverless environment + +### Dashboard vs Application Page Behavior + +**Dashboard (Sequential API calls):** +- Makes visits API call first, then views API call ~42 seconds later +- Both calls hit same server instance, cache ages show expected 42-second difference +- **This 42-second difference is normal and expected behavior** + +**Application page (Parallel API calls):** +- Makes both API calls simultaneously using `Promise.all()` +- Calls may hit different serverless instances due to load balancing +- Both instances access the same shared cached data +- Small age differences (1-2 seconds) are normal network/processing delays + +### Key Insights + +1. **42-second age differences** in Dashboard logs are **expected** due to sequential API calls +2. **1-2 second age differences** in application page logs are **normal** network delays +3. **Different instance IDs** handling requests is **normal** Vercel load balancing +4. **Cache sharing across instances** works **perfectly** - this was a key validation ## Console Logging @@ -206,17 +248,16 @@ The caching system provides detailed console logs: ### Cache Hit (Valid): ``` -☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668400000 +☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456 ✅ Cache data valid, age: 45s ``` ### Cache Miss (Expired): ``` -☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668400000 +☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456 ⏰ Cache expired: age=350s, ttl=300s -🔄 Cache buster updated: 1731668500000 -☁️ Using unstable_cache for Vercel: visits_abc123_vdpl_def456_1731668500000 -🔥 unstable_cache MISS - executing API call +🔄 Cache data expired, fetching fresh data +🆕 Making fresh API call due to expired cache ``` ### Deployment Invalidation: @@ -349,10 +390,10 @@ No additional configuration required. ### How It Works - Uses Next.js `unstable_cache` for persistence across serverless function instances -- **Three-layer cache invalidation strategy**: +- **Deployment-based cache invalidation strategy**: 1. **Deployment versioning**: Cache keys include `VERCEL_DEPLOYMENT_ID` - automatically invalidates on deploy - 2. **Application-layer TTL**: Validates cached data timestamp (2 minutes) regardless of Next.js cache - 3. **Manual cache-busting**: Updates timestamp in cache keys when user clicks "Clear Cache" + 2. **Application-layer TTL**: Validates cached data timestamp (5 minutes) regardless of Next.js cache + 3. **Manual cache clearing**: Limited effectiveness - uses Next.js revalidation APIs - Cached data includes timestamp for application-layer validation - **Removes `revalidate` parameter** - Next.js `revalidate` doesn't work reliably on Vercel @@ -361,27 +402,26 @@ No additional configuration required. - **Cache Persistence**: Data may persist in Next.js cache, but application validates age - **Cache Invalidation**: - Automatic: New deployments use different cache keys (`VERCEL_DEPLOYMENT_ID` changes) - - Time-based: Application checks `cachedAt` timestamp and refetches if > 2 minutes old - - Manual: "Clear Cache" button updates cache-buster timestamp + - Time-based: Application checks `cachedAt` timestamp and refetches if > 5 minutes old + - Manual: Limited effectiveness - "Clear Cache" uses Next.js revalidation APIs ### Cache Key Format ``` -endpoint_hash_vDEPLOYMENT_ID_timestamp -// Example: views_a1b2c3d4_vdpl_abc123_1728854400000 +endpoint_hash_vDEPLOYMENT_ID +// Example: views_a1b2c3d4_vdpl_abc123 ``` **Key Components**: - `endpoint_hash`: Base cache key from request parameters - `vDEPLOYMENT_ID`: Vercel deployment ID (changes with each deploy) -- `timestamp`: Cache-buster timestamp (updated on manual invalidation) ## API Routes with Caching All Acquia API routes use the hybrid caching system: -- **`/api/acquia/views`** - Views data with 6-hour cache -- **`/api/acquia/visits`** - Visits data with 6-hour cache -- **`/api/acquia/applications`** - Application data with 6-hour cache +- **`/api/acquia/views`** - Views data with 5-minute cache +- **`/api/acquia/visits`** - Visits data with 5-minute cache +- **`/api/acquia/applications`** - Application data with 5-minute cache ### Cache Key Generation @@ -420,13 +460,13 @@ Both the Dashboard and application detail pages include "Clear Cache" buttons th } ``` -**`DELETE /api/cache`** - Clear all cached data +**`DELETE /api/cache`** - Attempt to clear cached data ```json { - "message": "Cache cleared successfully", - "environment": "local" | "vercel", - "method": "file-clear" | "cache-buster", - "timestamp": "2025-10-10T21:52:35.743Z" + "success": true, + "environment": "vercel", + "method": "deployment-based", + "note": "Cache will be cleared on next deployment" } ``` @@ -443,16 +483,16 @@ Both the Dashboard and application detail pages include "Clear Cache" buttons th - ✅ **Cross-instance persistence** (shared across all serverless functions) - ✅ **Automatic deployment invalidation** (new `VERCEL_DEPLOYMENT_ID` = new cache keys) - ✅ **Consistent TTL behavior** (application validates timestamps) -- ✅ **Manual cache clearing** (cache-busting updates timestamp) +- ⚠️ **Limited manual cache clearing** (relies on Next.js revalidation APIs) - ⚠️ **Old cache may persist in Next.js layer** (but app won't use it if expired) ## Cache Duration -All cached data has a **2-minute TTL**: +All cached data has a **5-minute TTL**: ```typescript // Validated at application layer, not relying on Next.js revalidate -const CACHE_TTL_MS = 2 * 60 * 1000; // 2 minutes in milliseconds +const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes in milliseconds // Cached data includes timestamp { @@ -463,16 +503,17 @@ const CACHE_TTL_MS = 2 * 60 * 1000; // 2 minutes in milliseconds // Age validation on every request if (!isCacheDataValid(cachedResult.cachedAt)) { - // Refetch if older than 2 minutes - await updateCacheBuster(); - return getCachedApiData(apiCall, cacheKey, tags); + // Refetch if older than 5 minutes + console.log('🆕 Making fresh API call due to expired cache'); + const freshResult = await apiCall(); + return freshResult; } ``` **Why application-layer validation?** - Next.js `revalidate` parameter doesn't work reliably on Vercel -- Testing showed cached data persisting for 2+ hours despite `revalidate: 120` -- Application-layer timestamp checks ensure consistent 2-minute TTL behavior +- Testing showed cached data persisting for 2+ hours despite `revalidate: 300` +- Application-layer timestamp checks ensure consistent 5-minute TTL behavior ## Debugging and Monitoring @@ -492,26 +533,26 @@ The caching system provides detailed console logging: ``` 🏠 Using file cache for local development: views_abc123 🔥 File cache MISS - executing API call: views_abc123 -☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 +☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456 📦 Cache version (deployment): dpl_def456 -🔄 Cache buster: 1728854400000 -⏰ Cache expired: age=150s, ttl=120s +⏰ Cache expired: age=350s, ttl=300s ✅ Cache data valid, age: 45s -🗑️ Cache buster updated: 1728854500000 +🗑️ Revalidated tag: acquia-api ``` ### Log Prefixes - 🏠 Local development operations - ☁️ Vercel/production operations - 🔥 Cache miss (fresh API call) -- 📦 Cache hit (served from cache) -- 🗑️ Cache clearing operations +- ✅ Cache hit (served from cache) +- 🗑️ Cache clearing operations (limited effectiveness in production) - 🔍 Environment detection - 🗝️ Cache key generation +- 📦 Deployment versioning ## Troubleshooting -### Cache Not Expiring After 2 Minutes +### Cache Not Expiring After 5 Minutes 1. **Check console logs for timestamp validation**: - Look for "⏰ Cache expired" or "✅ Cache data valid, age: Xs" - Verify age is being calculated correctly @@ -557,10 +598,10 @@ The caching system provides detailed console logging: ### For Cache Duration -1. **2 minutes is appropriate** for most Acquia analytics data while keeping it relatively fresh +1. **5 minutes is appropriate** for most Acquia analytics data while balancing performance with freshness 2. **Application-layer validation ensures consistent behavior** across all environments 3. **Deployment-based versioning** automatically invalidates stale cache on deploy -4. **Manual clearing always available** for immediate updates when needed +4. **Manual clearing has limited effect** - mainly uses Next.js revalidation APIs 5. **Adjust `CACHE_TTL_MS`** in `lib/cache-hybrid.ts` if different TTL is needed ## Implementation Details @@ -568,11 +609,11 @@ The caching system provides detailed console logging: ### Why This Approach? **Problem**: Next.js `unstable_cache` with `revalidate` parameter doesn't work reliably on Vercel: -- Testing showed cached data persisting for 2+ hours despite `revalidate: 120` (2 minutes) +- Testing showed cached data persisting for 2+ hours despite `revalidate: 300` (5 minutes) - Cache persisted across deployments and manual invalidation attempts - `revalidateTag()` and `revalidatePath()` had no effect -**Solution**: Three-layer cache invalidation: +**Solution**: Deployment-based cache invalidation with application-layer validation: 1. **Deployment Versioning** (automatic): - Cache keys include `VERCEL_DEPLOYMENT_ID` or `VERCEL_GIT_COMMIT_SHA` @@ -583,24 +624,62 @@ The caching system provides detailed console logging: ```typescript // Check timestamp on every request if (!isCacheDataValid(cachedResult.cachedAt)) { - // Fetch fresh data if > 2 minutes old - await updateCacheBuster(); - return getCachedApiData(apiCall, cacheKey, tags); + // Fetch fresh data if > 5 minutes old + console.log('🆕 Making fresh API call due to expired cache'); + const freshResult = await apiCall(); + return freshResult; } ``` - Validates `cachedAt` timestamp in application code - Doesn't rely on Next.js cache behavior - - Consistent 2-minute TTL guaranteed + - Consistent 5-minute TTL guaranteed -3. **Manual Cache-Busting** (user-triggered): - - "Clear Cache" button updates `cacheBusterTimestamp` - - New requests use new cache keys - - Immediate invalidation for all subsequent requests +3. **Manual Cache Clearing** (limited effectiveness): + - "Clear Cache" button attempts to use Next.js revalidation APIs + - Limited effectiveness due to Vercel serverless architecture + - Cache automatically clears on next deployment ### Advantages - ✅ **Predictable TTL**: Application controls expiration, not Next.js - ✅ **Automatic deployment invalidation**: No stale cache after deploys -- ✅ **Manual control**: Users can force refresh when needed +- ✅ **Simplified cache keys**: Deployment-only versioning reduces complexity - ✅ **Cross-instance consistency**: All instances respect timestamp validation -- ✅ **Debugging**: Clear console logs show cache version, age, and validation \ No newline at end of file +- ✅ **Clear debugging**: Console logs show cache version, age, and validation +- ⚠️ **Limited manual clearing**: Cache clearing relies on Next.js APIs with limited effectiveness + +## Implementation Success Summary + +### Problems Solved ✅ + +1. **Cache TTL Reliability**: 5-minute server-side cache works consistently +2. **Cross-Instance Sharing**: Different serverless instances successfully share cached data +3. **Browser Cache Interference**: Eliminated through `cache: 'reload'` and unique `t=` parameters +4. **Deployment Cache Invalidation**: Automatic via deployment-based cache keys +5. **Age Calculation Accuracy**: Timestamp validation works correctly across instances + +### Key Validations + +**From Vercel Production Logs:** +``` +📊 Instance 1xk162: Retrieved cached data from instance 4czmjg, cached at: 2025-11-14T23:20:31.122Z +✅ Instance 1xk162: Cache data valid, age: 135s +``` + +**This proves the final implementation works as designed:** +- Instance A can read cache created by Instance B +- Age calculations are accurate across instances +- Cache sharing works in Vercel's serverless environment +- 42-second age differences between APIs are expected (sequential calls) +- 1-2 second differences are normal network delays (parallel calls) + +### Final Architecture + +- **Local Development**: File-based cache (`lib/cache.ts`) with 5-minute TTL +- **Vercel Production**: `unstable_cache` with deployment-based keys + timestamp validation +- **Cache Duration**: 5 minutes enforced at application layer +- **Browser Cache**: Completely disabled via headers and unique URLs +- **Manual Clearing**: Limited effectiveness, automatic clearing on deployment +- **Cross-Instance**: Proven to work reliably across Vercel serverless functions + +The caching implementation is **production-ready** and performs optimally! 🎉 \ No newline at end of file diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index b8b382b..ed62323 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -17,9 +17,6 @@ function getCacheVersion(): string { return deploymentId; } -// Generate a unique instance ID for debugging -const instanceId = Math.random().toString(36).substring(2, 8); - // Get the current cache buster timestamp async function getCacheBuster(): Promise { if (cacheBusterTimestamp === null) { @@ -76,25 +73,21 @@ export async function getCachedApiData( const cacheVersion = getCacheVersion(); const versionedCacheKey = `${cacheKey}_v${cacheVersion}`; - console.log(`☁️ Instance ${instanceId}: Using unstable_cache for Vercel: ${versionedCacheKey}`); - console.log(`🏷️ Instance ${instanceId}: Cache tags: ${tags.join(', ')}`); - console.log(`📦 Instance ${instanceId}: Cache version (deployment): ${cacheVersion}`); + console.log(`☁️ Using unstable_cache for Vercel: ${versionedCacheKey}`); + console.log(`🏷️ Cache tags: ${tags.join(', ')}`); + console.log(`📦 Cache version (deployment): ${cacheVersion}`); const cachedCall = unstable_cache( async () => { - console.log(`🔥 Instance ${instanceId}: unstable_cache MISS - executing API call: ${versionedCacheKey}`); + console.log(`🔥 unstable_cache MISS - executing API call: ${versionedCacheKey}`); const result = await apiCall(); // Wrap the result with timestamp for validation - const cachedData = { + return { data: result, cachedAt: new Date().toISOString(), - cacheKey: versionedCacheKey, - instanceId: instanceId + cacheKey: versionedCacheKey }; - - console.log(`💾 Instance ${instanceId}: Caching new data with timestamp: ${cachedData.cachedAt}`); - return cachedData; }, [versionedCacheKey], { @@ -106,18 +99,16 @@ export async function getCachedApiData( const cachedResult = await cachedCall(); - console.log(`📊 Instance ${instanceId}: Retrieved cached data from instance ${cachedResult.instanceId || 'unknown'}, cached at: ${cachedResult.cachedAt}`); - // Validate cache age at application level if (!isCacheDataValid(cachedResult.cachedAt)) { - console.log(`🔄 Instance ${instanceId}: Cache data expired, fetching fresh data`); - console.log(`⏰ Instance ${instanceId}: Cache was created at: ${cachedResult.cachedAt}`); - console.log(`⏰ Instance ${instanceId}: Current time: ${new Date().toISOString()}`); - console.log(`⏰ Instance ${instanceId}: Cache age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); + console.log(`🔄 Cache data expired, fetching fresh data`); + console.log(`⏰ Cache was created at: ${cachedResult.cachedAt}`); + console.log(`⏰ Current time: ${new Date().toISOString()}`); + console.log(`⏰ Cache age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); // For expired cache, we need to force a fresh API call // Since we can't modify unstable_cache keys dynamically, we'll just call the API directly - console.log(`🆕 Instance ${instanceId}: Making fresh API call due to expired cache`); + console.log(`🆕 Making fresh API call due to expired cache`); const freshResult = await apiCall(); // Note: We can't easily update the existing cache entry, but that's okay @@ -125,7 +116,7 @@ export async function getCachedApiData( return freshResult; } - console.log(`✅ Instance ${instanceId}: Cache data valid, age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); + console.log(`✅ Cache data valid, age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); return cachedResult.data; } } From ea9e661897940e6af9415c18069f1c7392d82784 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 16:14:27 -0800 Subject: [PATCH 30/44] Repairs to /applications landing page --- app/applications/page.tsx | 342 ++++++++++++++++++++++++------------ components/CountUpTimer.tsx | 6 +- 2 files changed, 228 insertions(+), 120 deletions(-) diff --git a/app/applications/page.tsx b/app/applications/page.tsx index 192ac3d..86541ca 100644 --- a/app/applications/page.tsx +++ b/app/applications/page.tsx @@ -1,9 +1,48 @@ 'use client'; import React, { useState, useEffect } from 'react'; +import CountUpTimer from '@/components/CountUpTimer'; + +const DEFAULT_SUBSCRIPTION_UUID = process.env.NEXT_PUBLIC_ACQUIA_SUBSCRIPTION_UUID || ""; + +// Get current Pacific time formatted string +function getCurrentPacificTimeString() { + const now = new Date(); + const pacificTime = now.toLocaleString('en-US', { + timeZone: 'America/Los_Angeles', + month: 'long', + day: 'numeric', + year: 'numeric', + hour: 'numeric', + minute: '2-digit', + timeZoneName: 'short' + }); + return pacificTime; +} async function fetchData(cacheBuster?: string) { - const params = new URLSearchParams(); + const subscriptionUuid = DEFAULT_SUBSCRIPTION_UUID; + + if (!subscriptionUuid) { + throw new Error('Subscription UUID is required. Please check your environment configuration.'); + } + + const params = new URLSearchParams({ + subscriptionUuid, + }); + + // Only add date parameters if we want to filter (comment out for now to match Dashboard behavior) + // // Default to current month data (like the Dashboard) + // const now = new Date(); + // const startOfMonth = new Date(now.getFullYear(), now.getMonth(), 1); + // + // // Convert to ISO 8601 format with time components as required by Acquia API + // const defaultFrom = startOfMonth.toISOString(); // Full ISO format: 2025-11-01T08:00:00.000Z + // const defaultTo = now.toISOString(); // Full ISO format: 2025-11-14T23:59:59.000Z + // + // params.append('from', defaultFrom); + // params.append('to', defaultTo); + if (cacheBuster) { params.set('t', cacheBuster); } @@ -19,18 +58,49 @@ async function fetchData(cacheBuster?: string) { }, }; + console.log('📱 Fetching applications with URL:', `/api/acquia/applications${suffix}`); + console.log('📊 Fetching views with URL:', `/api/acquia/views${suffix}`); + console.log('📈 Fetching visits with URL:', `/api/acquia/visits${suffix}`); + console.log('🔧 Environment check:', { + hasSubscriptionUuid: !!DEFAULT_SUBSCRIPTION_UUID, + subscriptionUuidLength: DEFAULT_SUBSCRIPTION_UUID?.length || 0, + noDateFilter: true // Fetching all available data like Dashboard + }); + const [appsRes, viewsRes, visitsRes] = await Promise.all([ fetch(`/api/acquia/applications${suffix}`, fetchOptions), fetch(`/api/acquia/views${suffix}`, fetchOptions), fetch(`/api/acquia/visits${suffix}`, fetchOptions), ]); + + // Check for HTTP errors + if (!appsRes.ok) { + const errorText = await appsRes.text(); + console.error('❌ Applications API Error:', { status: appsRes.status, statusText: appsRes.statusText, response: errorText }); + throw new Error(`Failed to fetch applications: ${appsRes.status} ${appsRes.statusText} - ${errorText}`); + } + if (!viewsRes.ok) { + const errorText = await viewsRes.text(); + console.error('❌ Views API Error:', { status: viewsRes.status, statusText: viewsRes.statusText, response: errorText }); + throw new Error(`Failed to fetch views: ${viewsRes.status} ${viewsRes.statusText} - ${errorText}`); + } + if (!visitsRes.ok) { + const errorText = await visitsRes.text(); + console.error('❌ Visits API Error:', { status: visitsRes.status, statusText: visitsRes.statusText, response: errorText }); + throw new Error(`Failed to fetch visits: ${visitsRes.status} ${visitsRes.statusText} - ${errorText}`); + } + const [apps, viewsRaw, visitsRaw] = await Promise.all([ - appsRes.ok ? appsRes.json() : [], - viewsRes.ok ? viewsRes.json() : [], - visitsRes.ok ? visitsRes.json() : [], + appsRes.json(), + viewsRes.json(), + visitsRes.json(), ]); - // Defensive: ensure arrays + console.log('📱 Applications response:', { length: apps?.length, sample: apps?.[0] }); + console.log('📊 Views response:', { length: viewsRaw?.data?.length || viewsRaw?.length, type: typeof viewsRaw }); + console.log('📈 Visits response:', { length: visitsRaw?.data?.length || visitsRaw?.length, type: typeof visitsRaw }); + + // Handle the response format from Acquia API const views = Array.isArray(viewsRaw) ? viewsRaw : viewsRaw && Array.isArray(viewsRaw.data) @@ -41,25 +111,69 @@ async function fetchData(cacheBuster?: string) { : visitsRaw && Array.isArray(visitsRaw.data) ? visitsRaw.data : []; + return { apps, views, visits }; } -function getAppStats(apps: any[], views: { map: (arg0: (v: any) => any[]) => Iterable; reduce: (arg0: (sum: any, v: any) => any, arg1: number) => any; }, visits: { map: (arg0: (v: any) => any[]) => Iterable; reduce: (arg0: (sum: any, v: any) => any, arg1: number) => any; }) { - // Map views/visits by app uuid - const viewsByApp = Object.fromEntries(views.map(v => [v.uuid, v.views])); - const visitsByApp = Object.fromEntries(visits.map(v => [v.uuid, v.visits])); - // Calculate totals - const totalViews = views.reduce((sum, v) => sum + v.views, 0); - const totalVisits = visits.reduce((sum, v) => sum + v.visits, 0); - - // Merge stats - return apps.map(app => ({ - ...app, - views: viewsByApp[app.uuid] || 0, - visits: visitsByApp[app.uuid] || 0, - viewsPct: totalViews ? ((viewsByApp[app.uuid] || 0) / totalViews) * 100 : 0, - visitsPct: totalVisits ? ((visitsByApp[app.uuid] || 0) / totalVisits) * 100 : 0, - })); +// Helper function to aggregate application statistics +function getAppStats(apps: any[], views: any[], visits: any[]) { + // UUIDs to exclude from the applications list + const EXCLUDED_UUIDS = [ + '2b2d2517-3839-414e-85a4-7183adc22283', + '1ef402a7-c301-42d7-9b63-f226fa1b2329' + ]; + + // Filter out excluded applications + const filteredApps = apps.filter(app => !EXCLUDED_UUIDS.includes(app.uuid)); + + console.log(`🔍 Filtered applications: ${apps.length} -> ${filteredApps.length} (excluded ${apps.length - filteredApps.length} apps)`); + + // Create summaries by application UUID + const viewsByApp: Record = {}; + const visitsByApp: Record = {}; + + // Sum views by application UUID (only for non-excluded apps) + views.forEach(record => { + const uuid = record.applicationUuid; + if (uuid && !EXCLUDED_UUIDS.includes(uuid)) { + viewsByApp[uuid] = (viewsByApp[uuid] || 0) + (record.views || 0); + } + }); + + // Sum visits by application UUID (only for non-excluded apps) + visits.forEach(record => { + const uuid = record.applicationUuid; + if (uuid && !EXCLUDED_UUIDS.includes(uuid)) { + visitsByApp[uuid] = (visitsByApp[uuid] || 0) + (record.visits || 0); + } + }); + + // Calculate totals for percentage calculations (from all non-excluded data) + const totalViews = Object.values(viewsByApp).reduce((sum, views) => sum + views, 0); + const totalVisits = Object.values(visitsByApp).reduce((sum, visits) => sum + visits, 0); + + console.log('📊 Data totals:', { totalViews, totalVisits, appsWithData: Object.keys(viewsByApp).length }); + + // Generate stats for each filtered application + const stats = filteredApps.map(app => { + const views = viewsByApp[app.uuid] || 0; + const visits = visitsByApp[app.uuid] || 0; + const viewsPct = totalViews > 0 ? (views / totalViews) * 100 : 0; + const visitsPct = totalVisits > 0 ? (visits / totalVisits) * 100 : 0; + + return { + uuid: app.uuid, + name: app.name || `App ${app.uuid.substring(0, 8)}`, + views, + visits, + viewsPct, + visitsPct + }; + }).filter(app => app.views > 0 || app.visits > 0) // Only include apps with data + .sort((a, b) => (b.views + b.visits) - (a.views + a.visits)); // Sort by total activity + + console.log('📊 Generated stats for', stats.length, 'applications'); + return stats; } export default function ApplicationsPage() { @@ -69,18 +183,20 @@ export default function ApplicationsPage() { const [views, setViews] = useState([]); const [visits, setVisits] = useState([]); const [stats, setStats] = useState([]); - const [cacheClearing, setCacheClearing] = useState(false); + const [elapsedTime, setElapsedTime] = useState(null); const loadData = async () => { setLoading(true); setError(null); + setElapsedTime(null); + + const startTime = Date.now(); try { console.log('📊 Fetching applications data with cache-busting parameter'); const { apps, views, visits } = await fetchData(Date.now().toString()); - const calculatedStats = getAppStats(apps, views, visits); - setApps(apps); + const calculatedStats = getAppStats(apps, views, visits); setApps(apps); setViews(views); setVisits(visits); setStats(calculatedStats); @@ -89,96 +205,64 @@ export default function ApplicationsPage() { } catch (err) { console.error('❌ Failed to load applications data:', err); const errorMessage = err instanceof Error ? err.message : 'An error occurred while fetching data'; + console.error('❌ Full error details:', { + error: err, + stack: err instanceof Error ? err.stack : undefined, + message: errorMessage, + type: typeof err, + name: err instanceof Error ? err.name : undefined + }); setError(errorMessage); } finally { + const endTime = Date.now(); + const timeElapsed = (endTime - startTime) / 1000; + setElapsedTime(timeElapsed); setLoading(false); } }; - const clearCache = async () => { - setCacheClearing(true); - try { - console.log('🗑️ Attempting to clear cache...'); - - // Clear browser cache first - if ('caches' in window) { - const cacheNames = await caches.keys(); - await Promise.all( - cacheNames.map(cacheName => caches.delete(cacheName)) - ); - console.log('🗑️ Cleared browser caches:', cacheNames); - } - - // Clear server cache - const response = await fetch('/api/cache', { method: 'DELETE' }); - - if (response.ok) { - const result = await response.json(); - console.log('✅ Server cache cleared:', result); - - const environment = result.environment || 'unknown'; - const method = result.method || 'unknown'; - - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); - - // Reload data with fresh cache-busting parameter - await loadData(); - } else { - const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); - console.error('❌ Failed to clear cache:', errorData); - alert(`Failed to clear cache: ${errorData.error || 'Unknown error'}`); - } - } catch (error) { - console.error('❌ Cache clearing error:', error); - alert(`Error clearing cache: ${error instanceof Error ? error.message : 'Network error'}`); - } finally { - setCacheClearing(false); - } - }; - // Load data on component mount useEffect(() => { loadData(); }, []); return ( -
-
-

Application Views & Visits

- -
- - - +
+
+
+

Application Views & Visits

+
+ Current month-to-date data (as of {getCurrentPacificTimeString()}) +
{loading && ( -
-
Loading applications data...
+
+
+ +
Loading applications data...
+
+
+ )} + + {!loading && elapsedTime !== null && ( +
+
+ +
+ Data loaded in {elapsedTime?.toFixed(1) || '0.0'} seconds +
+
)} {error && ( -
-
Error: {error}
+
+
Error: {error}
@@ -186,35 +270,59 @@ export default function ApplicationsPage() { )} {!loading && !error && stats.length > 0 && ( - - - - - - - - - - - - - {stats.map(app => ( - - - - - - - +
+
ApplicationUUIDViews% of ViewsVisits% of Visits
{app.name}{app.uuid}{app.views.toLocaleString()}{app.viewsPct.toFixed(1)}%{app.visits.toLocaleString()}{app.visitsPct.toFixed(1)}%
+ + + + + + + + - ))} - -
ApplicationUUIDViews% of ViewsVisits% of Visits
+ + + {stats.map((app, index) => ( + + + + {app.name} + + + {app.uuid} + + {app.views.toLocaleString()} + + {app.viewsPct.toFixed(1)}% + + {app.visits.toLocaleString()} + + {app.visitsPct.toFixed(1)}% + + ))} + + +
)} {!loading && !error && stats.length === 0 && ( -
-
No application data found
+
+
No application data found
+
+ This could be due to: +
    +
  • Missing subscription UUID in environment configuration
    • +
  • No data available for the current time period
    • +
  • API connection issues
    • +
+
+
+ Subscription UUID: {DEFAULT_SUBSCRIPTION_UUID || 'NOT_SET'} +
)}
diff --git a/components/CountUpTimer.tsx b/components/CountUpTimer.tsx index 0368b78..bc52403 100644 --- a/components/CountUpTimer.tsx +++ b/components/CountUpTimer.tsx @@ -39,7 +39,7 @@ const CountUpTimer: React.FC = ({ isRunning, finalTime }) => clearInterval(timerIdRef.current); timerIdRef.current = null; } - + // If a final time is provided, display it. Otherwise, keep the last calculated time. if (finalTime !== undefined && finalTime !== null) { setElapsedTime(finalTime); @@ -60,8 +60,8 @@ const CountUpTimer: React.FC = ({ isRunning, finalTime }) => const formattedTime = elapsedTime.toFixed(1); return ( -
- {formattedTime}s +
+ {formattedTime}s
); }; From 7a082e294190d7aaa08cdb30bb3af9eb649e1e25 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 16:40:53 -0800 Subject: [PATCH 31/44] Add copilot instructions for ACHOO-119 branch --- .github/copilot-instructions.md | 297 ++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..7b23b02 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,297 @@ +# CHURRO - AI Development Guidelines + +## Project Overview +**CHURRO** (Cloud Hosting Usage Reporting with Recurring Output) is a Next.js 15 dashboard for visualizing Acquia Cloud hosting analytics (views/visits data). Built for Stanford University with Stanford Design System (Decanter) styling and basic HTTP authentication. + +## Architecture + +### Tech Stack +- **Framework**: Next.js 15.5+ (App Router only, no Pages Router) +- **Runtime**: Node.js 22.x (enforced via package.json engines) +- **Styling**: TailwindCSS with Decanter preset (Stanford Design System) +- **Authentication**: Basic HTTP Authentication via middleware +- **Data Viz**: Recharts for charts/graphs +- **Deployment**: Vercel + +### Key Components + +**API Routes** (`app/api/`): +- `/api/acquia/applications` - Fetches all Acquia applications +- `/api/acquia/visits` - Fetches visits metrics with pagination +- `/api/acquia/views` - Fetches views metrics with pagination +- `/api/cache` - Cache management endpoint (GET/DELETE) +- `/api/debug-env` - Environment variable debugging + +**Core Services** (`lib/`): +- `lib/acquia-api.ts` - Acquia Cloud API client with hybrid caching and pagination +- `lib/cache-hybrid.ts` - Hybrid caching system (file-based local, unstable_cache production) +- `lib/cache.ts` - File-based caching for local development + +**Main Pages**: +- `/` - Dashboard with date filtering and tabbed views (Dashboard component) +- `/applications` - Applications overview table (auto-loading, filtered) +- `/applications/[uuid]` - Individual application detail with daily charts + +**Data Flow**: +1. User accesses application → Basic auth via middleware +2. Dashboard/pages fetch from `/api/acquia/*` routes +3. API routes use `AcquiaApiServiceFixed` with OAuth2 client_credentials flow +4. Response data parsed from Acquia's nested `_embedded.items[].datapoints[]` structure +5. Data cached using hybrid cache system for 5 minutes + +## Development Patterns + +### Environment Variables +**Required** (stored in `.env.local`, never committed): +- `ACQUIA_API_KEY` / `ACQUIA_API_SECRET` - OAuth2 credentials (no quotes) +- `NEXT_PUBLIC_ACQUIA_SUBSCRIPTION_UUID` - Subscription identifier +- `NEXT_PUBLIC_ACQUIA_MONTHLY_{VIEWS|VISITS}_ENTITLEMENT` - Usage limits +- `ACQUIA_API_BASE_URL` - API base URL (defaults to https://cloud.acquia.com/api) +- `ACQUIA_AUTH_BASE_URL` - Auth base URL (defaults to https://accounts.acquia.com/api) + +**Critical**: Values must NOT have surrounding quotes. API key/secret are auto-stripped of quotes in `acquia-api.ts`. + +### Authentication + +**Basic HTTP Authentication** (`middleware.ts`): +- Username: `sws` +- Password: `sws` +- Protects all routes except `/_next`, `/api/public`, `/favicon.ico` +- Allows localhost access (IPv4/IPv6) without authentication +- Returns 401 with WWW-Authenticate header for protected resources + +### Stanford Design System (Decanter) + +**Colors** - Use semantic Decanter tokens, NOT hex values: +- Primary: `cardinal-red` (Stanford brand red) +- Accents: `digital-blue`, `digital-green` +- Grays: `black-{10,20,60,80}`, `gc-black` +- **Never use**: Custom hex colors unless explicitly non-Stanford branding + +**Typography**: +- Headings: `type-{0,1,2,3,4}` classes (defined in Decanter) +- Body: `Source_Sans_3` (imported in `app/layout.tsx`) +- Serif: `Source_Serif_4` for emphasis +- Display: Custom `stanford` font (local woff2) + +**Interaction States**: +- Use `hocus:` prefix for hover+focus states (Decanter utility) +- Example: `hocus:bg-black hocus:text-white` + +**Spacing**: +- Use Decanter scale: `p-{5,8,10,15,20,25,30,50}` (not arbitrary values) +- Responsive: `px-20 sm:px-30 md:px-50 lg:px-30` + +### Acquia API Integration + +**Authentication** (`lib/acquia-api.ts`): +- OAuth2 client credentials flow with 3 fallback methods +- Token cached in `this.accessToken`, auto-retries on 401 +- Credentials auto-cleaned of quotes + +**Data Parsing**: +- Response structure: `_embedded.items[]` where each item = one application +- Each item has `metadata.application.uuids[0]` + `datapoints[]` array +- Datapoints format: `["2025-04-15T00:00:00+00:00", "1124"]` (date, value) +- **Critical**: ONE item = ONE application, ALL datapoints belong to that app + +**Caching**: +- Hybrid cache system: file-based (local) vs unstable_cache (production) +- 5-minute cache duration with deployment-based invalidation +- Cache keys based on deployment ID for automatic invalidation +- Browser cache prevention via `cache: 'reload'` and unique timestamps + +**Date Filtering**: +- API expects ISO 8601: `2025-04-15T23:59:59.000Z` +- Frontend sends: `YYYY-MM-DD` (conditional - only if user sets dates) +- No default date filtering (fetches all available data) + +### Hybrid Caching System + +**Local Development** (`lib/cache.ts`): +- File-based caching in `.cache/` directory +- 5-minute TTL with file timestamps +- Persists across server restarts +- Manual clearing via file deletion + +**Production** (`lib/cache-hybrid.ts`): +- Next.js `unstable_cache` with deployment versioning +- Cache keys include `VERCEL_DEPLOYMENT_ID` for auto-invalidation +- Application-layer timestamp validation (5 minutes) +- Browser cache prevention via headers and unique URLs + +**Key Features**: +- Environment detection at runtime +- Deployment-based cache invalidation +- Cross-instance cache sharing on Vercel +- Manual cache clearing via `/api/cache` DELETE endpoint + +### Component Patterns + +**Client Components** - Use `'use client'` directive when: +- Using React hooks (`useState`, `useEffect`) +- Browser APIs (localStorage, fetch) +- Timer components (`CountUpTimer`) + +**Server Components** - Default for: +- Page layouts (`app/layout.tsx`) +- Static content rendering + +**Data Fetching**: +- Client-side: `fetch('/api/acquia/visits')` with cache-busting timestamps +- Pass query params: `new URLSearchParams({ subscriptionUuid, from, to })` +- Handle loading states with `CountUpTimer` component + +## Page Architecture + +### Dashboard (`/`) +- **Purpose**: Main analytics dashboard with date filtering +- **Features**: + - User-configurable date range inputs + - Subscription UUID input field + - Tabbed interface (pie charts, bar charts, data tables) + - Manual "Fetch Analytics Data" button + - Cache clearing functionality +- **Data**: Fetches all apps, views, and visits with optional date filtering + +### Applications Page (`/applications`) +- **Purpose**: Overview table of all applications with statistics +- **Features**: + - Auto-loads on page mount (no manual refresh needed) + - Shows current Pacific time timestamp + - Displays views/visits totals and percentages + - Links to individual application pages + - Excludes specific UUIDs: `2b2d2517-3839-414e-85a4-7183adc22283`, `1ef402a7-c301-42d7-9b63-f226fa1b2329` +- **Data**: Fetches all available data (no date filtering) +- **Styling**: Stanford Design System table with alternating rows + +### Application Detail (`/applications/[uuid]`) +- **Purpose**: Individual application analytics with daily resolution +- **Features**: + - Date range picker for custom filtering + - Daily resolution charts (line charts) + - Application name resolution from UUID + - Cache clearing functionality +- **Data**: Fetches daily data with `resolution=day` parameter + +## Common Tasks + +### Adding a New Chart Type +1. Create component in `components/` (e.g., `NewChart.tsx`) +2. Import Recharts primitives: `{ BarChart, Bar, XAxis, YAxis, Tooltip }` +3. Accept `data` prop with `{ name: string, value: number, uuid: string }[]` +4. Use Decanter colors: `className='fill-cardinal-red'` +5. Add tab to Dashboard TABS array +6. Add conditional render in tab content section + +### Debugging Acquia API Issues +1. Check environment variables via `/api/debug-env` +2. Verify auth credentials are set correctly (no quotes) +3. Check API route error responses for detailed error info +4. Monitor cache behavior via console logs +5. Use "Clear Cache" functionality to force fresh data + +### Environment Setup + +**Local Development**: +```bash +nvm use # Ensures Node 22.x +npm install # Install dependencies + +# Configure environment +cp .env.example .env.local # Create env file if needed +# Edit .env.local with required Acquia credentials + +npm run dev # Start development server +``` + +### Adding New Application Exclusions +1. Edit the `EXCLUDED_UUIDS` array in `/app/applications/page.tsx` +2. Add UUID string to the array +3. Applications will be filtered from both table display and statistics + +### Cache Management +1. **Local**: Cache stored in `.cache/` directory (gitignored) +2. **Production**: Cache cleared automatically on deployment +3. **Manual**: Use "Clear Cache" buttons in UI or `DELETE /api/cache` +4. **TTL**: 5 minutes across all environments + +## File Organization + +``` +app/ + api/ # API routes (Next.js 15 route handlers) + acquia/ # Acquia Cloud API proxy routes + cache/ # Cache management endpoint + debug-env/ # Environment debugging + applications/ # Applications pages + [uuid]/ # Individual application detail + page.tsx # Applications overview table + page.tsx # Home page (Dashboard component) + layout.tsx # Root layout with Stanford fonts +components/ # React components + Dashboard.tsx # Main dashboard with tabs and charts + CountUpTimer.tsx # Loading timer component + [Charts]/ # Recharts-based chart components + [UI]/ # Stanford Design System components +lib/ # Core business logic + acquia-api.ts # Acquia API client with OAuth2 and caching + cache-hybrid.ts # Hybrid caching system + cache.ts # File-based cache (local development) +docs/ # Documentation + caching.md # Comprehensive caching system docs +middleware.ts # Basic HTTP authentication +utilities/ # Helper utilities +``` + +## Testing & Verification + +**Local Testing**: +- Use basic auth credentials: sws/sws +- Test API endpoints directly via browser DevTools +- Check cache behavior via console logs +- Verify environment variables via `/api/debug-env` + +**Production Checklist**: +- Verify all environment variables set in Vercel +- Test basic authentication works +- Confirm caching behavior (5-minute TTL) +- Check that excluded applications don't appear in `/applications` + +## Common Pitfalls + +1. **Quoted env vars** - `ACQUIA_API_KEY="abc"` breaks auth (remove quotes) +2. **Wrong auth method** - This branch uses basic HTTP auth, not SAML +3. **Date format mismatch** - Frontend sends `YYYY-MM-DD`, API needs ISO 8601 +4. **Cache staleness** - 5-minute cache may hide API issues, use cache clearing +5. **Decanter overrides** - Don't use arbitrary Tailwind values, use Decanter tokens +6. **Application filtering** - Remember to add UUIDs to exclusion list when needed + +## Dependencies + +**Core Dependencies**: +- `next` ^15.5.2 - Next.js framework +- `react` 18.3.1 - React library +- `axios` ^1.12.0 - HTTP client for API calls +- `recharts` ^2.12.7 - Chart components +- `decanter` ^7.4.0 - Stanford Design System +- `basic-auth` ^2.0.1 - Basic authentication utilities + +**Development Dependencies**: +- `typescript` ^5.5.2 - TypeScript support +- `tailwindcss` ^3.4.17 - Utility-first CSS +- `eslint` ^8.57.0 - Code linting + +## Key Documentation + +- **Caching System**: `docs/caching.md` (comprehensive caching guide) +- **Acquia API**: https://cloud.acquia.com/api (official docs, requires login) +- **Decanter**: https://decanter.stanford.edu (Stanford Design System) +- **Next.js 15**: https://nextjs.org/docs (App Router only) + +## Branch Strategy +- Current branch: `ACHOO-119-20251015` +- Repository: `SU-SWS/churro` (Stanford University Web Services) + +--- +*Last Updated: November 2025 - Reflects current implementation with basic HTTP auth and hybrid caching* From ce6c5a455156e373b54ae4692f0e3118ab6aa917 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Fri, 14 Nov 2025 17:01:33 -0800 Subject: [PATCH 32/44] Remove working notes cache-invalidation-fix.md --- docs/cache-invalidation-fix.md | 392 --------------------------------- 1 file changed, 392 deletions(-) delete mode 100644 docs/cache-invalidation-fix.md diff --git a/docs/cache-invalidation-fix.md b/docs/cache-invalidation-fix.md deleted file mode 100644 index eb0d3f5..0000000 --- a/docs/cache-invalidation-fix.md +++ /dev/null @@ -1,392 +0,0 @@ -# Cache Invalidation Fix - Technical Summary - -## Problem Statement - -Testing on Vercel revealed that cached data was persisting far longer than the configured 2-minute TTL: - -``` -Time | Event ---------|------------------------------------------------------- -1:51 PM | Generated data from 10/01 to 10/21 -1:56 PM | All browsers still serving cached data (5 min - expected) -2:08 PM | All browsers still serving cached data (17 min - PROBLEM) -2:17 PM | Re-deployed application -2:18 PM | All browsers STILL serving cached data after deploy -3:38 PM | All browsers STILL serving cached data (nearly 2 hours!) -``` - -**Expected behavior**: Cache should expire after 2 minutes -**Actual behavior**: Cache persisted for 2+ hours, even across deployments - -## Root Cause Analysis - -The issue was caused by **multiple layers of caching**, not just Next.js `unstable_cache`: - -1. **`revalidate` parameter ignored**: Despite setting `revalidate: 120` (2 minutes), Vercel's cache layer was not respecting this value -2. **Persistent cache across deployments**: Cache survived deployments because cache keys didn't change -3. **Cache-buster insufficient**: The cache-buster timestamp was stored in a module-level variable that reset with each serverless function cold start, making it unreliable across instances -4. **Browser caching**: **Critical** - API routes were sending `Cache-Control: max-age=21600` (6 hours) headers, causing browsers to cache responses and never revalidate with the server - -### Why Next.js `revalidate` Doesn't Work on Vercel - -From Next.js documentation and community reports: -- `unstable_cache` is still experimental and behavior varies by environment -- Vercel's CDN and edge caching layers have their own persistence logic -- `revalidate` is a "suggestion" not a guarantee - Vercel may serve stale content longer -- Edge caching prioritizes performance over strict TTL adherence - -### Why Browser Caching Was the Real Culprit - -Even with server-side cache working correctly, **browser HTTP caching** was preventing fresh data: -- API routes were sending: `Cache-Control: public, max-age=21600` (6 hours) -- Browsers cached responses and served them without contacting the server -- Page refresh didn't help - browser returned cached response immediately -- Server-side cache validation never ran because requests never reached the server - -**Critical Discovery from Testing**: -- Even after changing to `max-age=120` (2 minutes), browser caching still interfered -- Browsers with old cached responses (6-hour TTL) continued serving stale data -- Hard refresh was required to bypass existing cache -- **Even `cache: 'no-store'` and `no-cache` headers didn't reliably prevent browser caching** -- **`cache: 'no-store'` doesn't force network requests** - browsers still return cached responses -- **Solution**: Use `cache: 'reload'` + timestamp query parameter that changes on every request to force unique URLs## Solution: Server-Side Cache Only (No Browser Caching) - -### Key Insight: Force Network Requests with Unique URLs - -After extensive testing, we discovered that **browser HTTP caching is extremely persistent** even with `no-store` directives. The only reliable solution is to **force unique URLs on every request** so the browser cannot use cached responses. - -**The approach**: -1. **Disable browser caching** with response headers (`no-store`) -2. **Add timestamp to every request** (`t=Date.now()`) - creates unique URL each time -3. **Server ignores timestamp** when generating cache keys - preserves server-side caching -4. **Use `cache: 'reload'`** in fetch options - forces network request - -**Result**: Browser always makes network request (unique URL), but server-side cache still works (ignores timestamp in cache key). - -### 0. Disable Browser Caching (Critical) - -**API routes** send headers that prevent ALL browser caching: - -```typescript -// In app/api/acquia/visits/route.ts and views/route.ts -response.headers.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate'); -response.headers.set('Pragma', 'no-cache'); -response.headers.set('Expires', '0'); -``` - -**Client fetch** requests with unique URLs on every request: - -```typescript -// In components/Dashboard.tsx -const params = new URLSearchParams({ - subscriptionUuid, - from: dateFrom, - to: dateTo, - t: Date.now().toString(), // Timestamp - changes on EVERY request, forcing unique URL -}); - -const fetchOptions: RequestInit = { - cache: 'reload', // Forces network request, bypassing browser cache - headers: { - 'Cache-Control': 'no-cache, no-store, must-revalidate', - 'Pragma': 'no-cache', - }, -}; -``` - -**API routes ignore the timestamp parameter**: - -```typescript -// In app/api/acquia/visits/route.ts and views/route.ts -const subscriptionUuid = searchParams.get('subscriptionUuid'); -const from = searchParams.get('from'); -const to = searchParams.get('to'); -const resolution = searchParams.get('resolution'); -// Note: t (timestamp) parameter is ignored - used only to force browser to make network request - -// Cache key only uses meaningful parameters -const cacheKey = generateApiCacheKey('visits', { - subscriptionUuid, - from, - to, - resolution - // t is NOT included - same cache key for same data request -}); -``` - -**Key changes**: -- `no-store` - Browser must not cache response at all -- `no-cache` - Browser must revalidate with server on every request -- `must-revalidate` - Browser cannot serve stale content -- `Pragma: no-cache` - HTTP/1.0 backwards compatibility -- **`cache: 'reload'`** - Forces network request (stronger than `no-store`) -- **`t` query parameter** - Changes on EVERY request, creating unique URL each time - -**Effect**: Every request goes to the server (unique URL = no cached response possible). Server checks cache age and returns cached data if < 2 minutes old. - -### 1. Deployment-Based Cache Versioning (Automatic) - -Cache keys now include Vercel's deployment ID, which changes with every deploy: - -```typescript -function getCacheVersion(): string { - const deploymentId = process.env.VERCEL_DEPLOYMENT_ID || - process.env.VERCEL_GIT_COMMIT_SHA?.substring(0, 8) || - 'local'; - return deploymentId; -} - -// Cache key: views_abc123_vdpl_def456_1728854400000 -// ^^^^^^^^^^^^^^ deployment ID -``` - -**Effect**: Every deployment automatically gets fresh cache keys, invalidating old cache - -### 2. Application-Layer Timestamp Validation (Automatic) - -Cached data now includes a timestamp that's validated on every request: - -```typescript -// Store with timestamp -const cachedData = { - data: result, - cachedAt: new Date().toISOString(), - cacheKey: versionedCacheKey -}; - -// Validate age on retrieval -const age = Date.now() - new Date(cachedResult.cachedAt).getTime(); -if (age >= CACHE_TTL_MS) { - // Refetch fresh data - await updateCacheBuster(); - return getCachedApiData(apiCall, cacheKey, tags); -} -``` - -**Effect**: Cache expires reliably after 2 minutes, regardless of Next.js cache behavior - -### 3. Manual Cache-Busting (User-Triggered) - -When users click "Clear Cache", updates the cache-buster timestamp: - -```typescript -export async function updateCacheBuster(): Promise { - const newTimestamp = Date.now(); - cacheBusterTimestamp = newTimestamp; - return newTimestamp; -} - -// New cache key: views_abc123_vdpl_def456_1728854500000 -// ^^^^^^^^^^^^^^ updated timestamp -``` - -**Effect**: Immediate cache invalidation for all subsequent requests - -### 4. Removed Unreliable `revalidate` Parameter - -```typescript -// BEFORE (didn't work on Vercel) -unstable_cache(apiCall, [cacheKey], { - revalidate: 120, // ❌ Ignored by Vercel - tags: ['acquia-api'] -}) - -// AFTER (application controls expiration) -unstable_cache(apiCall, [cacheKey], { - // No revalidate - we validate timestamps ourselves - tags: ['acquia-api'] -}) -``` - -## Code Changes - -### File: `lib/cache-hybrid.ts` - -**Added**: -- `CACHE_TTL_MS` constant (2 minutes in milliseconds) -- `getCacheVersion()` - Returns deployment ID for cache versioning -- `isCacheDataValid()` - Validates cached data age - -**Modified**: -- `getCachedApiData()`: - - Wraps cached data with `{ data, cachedAt, cacheKey }` - - Includes deployment version in cache keys - - Validates timestamp on every retrieval - - Recursively refetches if expired - - Removed `revalidate` parameter - -### Files: `app/api/acquia/visits/route.ts` and `app/api/acquia/views/route.ts` - -**Modified**: -- Response headers: `no-store, no-cache, must-revalidate, proxy-revalidate` -- Added `Pragma: no-cache` and `Expires: 0` for maximum compatibility -- **Completely disables browser caching** - all caching happens server-side - -### File: `components/Dashboard.tsx` - -**Modified**: -- Fetch requests include `t` query parameter with `Date.now()` timestamp -- Creates unique URL on every request to prevent browser caching -- Fetch options: Changed from `cache: 'no-store'` to `cache: 'reload'` -- Request headers: `Cache-Control: no-cache, no-store, must-revalidate` -- **Forces browser to always make network request** - unique URL bypasses any cached response -- **Server-side cache still works** - timestamp parameter ignored in cache key generation - -**Impact**: -- Cache now expires reliably after 2 minutes **on both server and browser** -- Automatic invalidation on deploy -- Browser actually makes requests to server after 2 minutes -- No breaking changes to API - -### File: `lib/cache.ts` - -**No changes required** - File-based cache for local development already working correctly - -### File: `docs/caching.md` - -**Updated sections**: -- Overview: Added deployment versioning and timestamp validation -- Cache behavior characteristics: Updated Vercel section -- Cache duration: Explained application-layer validation -- Console logging: Added new log examples -- Troubleshooting: Added timestamp validation checks -- Added new "Implementation Details" section - -## Testing Recommendations - -### Local Testing -```bash -npm run dev -# Open http://localhost:3000 -# Fetch data, wait 2+ minutes, refresh -# Should see new API call after 2 minutes -``` - -### Vercel Testing -After deploying: - -1. **Test automatic expiration**: - - Load page, note timestamp in console - - Wait 2 minutes - - Refresh page - - Check console for "⏰ Cache expired" and fresh API call - -2. **Test deployment invalidation**: - - Load page, note cache version in console - - Deploy new version - - Refresh page - - Cache version should change, forcing fresh data - -3. **Test manual clearing**: - - Load page - - Click "Clear Cache" button - - Refresh page - - Should see fresh API call immediately - -### Expected Console Logs - -**First request (cache miss)**: -``` -☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 -📦 Cache version (deployment): dpl_def456 -🔄 Cache buster: 1728854400000 -🔥 unstable_cache MISS - executing API call -``` - -**Second request within 2 minutes (cache hit)**: -``` -☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 -✅ Cache data valid, age: 45s -``` - -**Request after 2 minutes (expired)**: -``` -☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854400000 -⏰ Cache expired: age=150s, ttl=120s -🔄 Cache buster updated: 1728854500000 -☁️ Using unstable_cache for Vercel: views_abc123_vdpl_def456_1728854500000 -🔥 unstable_cache MISS - executing API call -``` - -## Environment Variables Used - -- `VERCEL_DEPLOYMENT_ID` (automatic on Vercel) - Primary deployment identifier -- `VERCEL_GIT_COMMIT_SHA` (automatic on Vercel) - Fallback deployment identifier -- `NODE_ENV` (set to 'production' on Vercel) - Environment detection -- `VERCEL_ENV` (automatic on Vercel) - Vercel environment detection - -No new environment variables required! - -## Performance Impact - -### Positive -- ✅ Faster subsequent requests (cache still works) -- ✅ No stale data after 2 minutes (reliable expiration) -- ✅ No stale data after deployments (automatic invalidation) -- ✅ Better cache hit rates (timestamp validation happens after cache lookup) - -### Neutral -- ⚠️ Minimal overhead from timestamp validation (< 1ms per request) -- ⚠️ Slightly longer cache keys due to deployment ID - -### No Negative Impact -- ✅ Same number of API calls to Acquia -- ✅ No additional database or storage requirements -- ✅ No changes to frontend code required - -## Future Considerations - -### If 2-Minute TTL Needs Adjustment - -Edit `CACHE_TTL_MS` in `lib/cache-hybrid.ts`: - -```typescript -// Current: 2 minutes -const CACHE_TTL_MS = 2 * 60 * 1000; - -// For 5 minutes: -const CACHE_TTL_MS = 5 * 60 * 1000; - -// For 1 hour: -const CACHE_TTL_MS = 60 * 60 * 1000; -``` - -### If Per-Endpoint TTL Needed - -Could pass TTL as parameter to `getCachedApiData()`: - -```typescript -export async function getCachedApiData( - apiCall: () => Promise, - cacheKey: string, - tags: string[] = [], - ttlMs: number = CACHE_TTL_MS // Add TTL parameter -): Promise -``` - -### If External Cache Needed - -For better cross-instance coordination, could use: -- Redis for cache-buster timestamp -- Vercel KV for shared state -- Database table for cache metadata - -Currently not needed - application-layer validation works across all instances. - -## Summary - -**Problem**: Vercel cache persisted 2+ hours despite 2-minute configuration -**Root causes**: -1. Next.js `revalidate` parameter not respected on Vercel -2. **Browser HTTP caching with 6-hour `max-age` preventing server requests** - -**Solution**: -1. Three-layer server approach (deployment versioning + timestamp validation + manual busting) -2. **Browser cache headers aligned with server TTL (2 minutes)** - -**Result**: Reliable 2-minute cache expiration on both server and browser + automatic deployment invalidation -**Impact**: No breaking changes, better cache behavior, comprehensive logging, **actual cache expiration**## References - -- Commit `6ab816e8aa157d52d23eb321a9408305dfb8f419` - Updated cache lifetime to 2 minutes -- Next.js `unstable_cache` docs: https://nextjs.org/docs/app/api-reference/functions/unstable_cache -- Vercel environment variables: https://vercel.com/docs/projects/environment-variables/system-environment-variables From 8909cbd7363f730c0db22e2d7a2a54ea1a775e39 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:08:51 -0800 Subject: [PATCH 33/44] Restore API key validation check --- app/api/acquia/views/route.ts | 18 ++++++++++++++++++ app/api/acquia/visits/route.ts | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 3a0c5ae..469d36f 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -19,6 +19,24 @@ export async function GET(request: NextRequest) { ); } + // Validate API credentials + if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { + console.error('❌ Missing required environment variables!'); + console.error('Available env vars:', Object.keys(process.env).filter(k => k.startsWith('ACQUIA'))); + return NextResponse.json( + { + error: 'Server configuration error: missing API credentials', + envCheck: { + ACQUIA_API_KEY: process.env.ACQUIA_API_KEY ? `${process.env.ACQUIA_API_KEY.substring(0, 8)}...` : 'missing', + ACQUIA_API_SECRET: process.env.ACQUIA_API_SECRET ? 'present' : 'missing', + ACQUIA_API_BASE_URL: process.env.ACQUIA_API_BASE_URL || 'missing', + ACQUIA_AUTH_BASE_URL: process.env.ACQUIA_AUTH_BASE_URL || 'missing' + } + }, + { status: 500 } + ); + } + // Generate cache key with ALL parameters const cacheKey = generateApiCacheKey('views', { subscriptionUuid, diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index 38cd1c8..fb67c98 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -19,6 +19,24 @@ export async function GET(request: NextRequest) { ); } + // Validate API credentials + if (!process.env.ACQUIA_API_KEY || !process.env.ACQUIA_API_SECRET) { + console.error('❌ Missing required environment variables!'); + console.error('Available env vars:', Object.keys(process.env).filter(k => k.startsWith('ACQUIA'))); + return NextResponse.json( + { + error: 'Server configuration error: missing API credentials', + envCheck: { + ACQUIA_API_KEY: process.env.ACQUIA_API_KEY ? `${process.env.ACQUIA_API_KEY.substring(0, 8)}...` : 'missing', + ACQUIA_API_SECRET: process.env.ACQUIA_API_SECRET ? 'present' : 'missing', + ACQUIA_API_BASE_URL: process.env.ACQUIA_API_BASE_URL || 'missing', + ACQUIA_AUTH_BASE_URL: process.env.ACQUIA_AUTH_BASE_URL || 'missing' + } + }, + { status: 500 } + ); + } + // Generate cache key with ALL parameters const cacheKey = generateApiCacheKey('visits', { subscriptionUuid, From d1d1004bc716183903d5a833a260a2dc7594afa2 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:10:59 -0800 Subject: [PATCH 34/44] Update alert message --- components/Dashboard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/Dashboard.tsx b/components/Dashboard.tsx index 555fab4..8db73e5 100644 --- a/components/Dashboard.tsx +++ b/components/Dashboard.tsx @@ -291,7 +291,7 @@ const Dashboard: React.FC = () => { const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches have been cleared and requests are designed to bypass browser caching.`); } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); From be9e0554edd9481f9de0b22a00afcf09a8125f49 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:12:09 -0800 Subject: [PATCH 35/44] Update alert message for uuid page --- app/applications/[uuid]/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/applications/[uuid]/page.tsx b/app/applications/[uuid]/page.tsx index adf2fff..7ee13c7 100644 --- a/app/applications/[uuid]/page.tsx +++ b/app/applications/[uuid]/page.tsx @@ -214,7 +214,7 @@ export default function ApplicationDetailPage({ params }: any) { const environment = result.environment || 'unknown'; const method = result.method || 'unknown'; - alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared\n\nNote: Browser may still have cached responses. Use hard refresh if needed.`); + alert(`Cache cleared successfully!\nEnvironment: ${environment}\nMethod: ${method}\nBrowser caches also cleared.\n\nIf you still see stale data, try refreshing the page.`); } else { const errorData = await response.json().catch(() => ({ error: 'Unknown error' })); console.error('❌ Failed to clear cache:', errorData); From db7467941f08784590b5a080e536dfe506996411 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:14:29 -0800 Subject: [PATCH 36/44] Update vercel.json to align with caching strategy --- vercel.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vercel.json b/vercel.json index 51c7ae6..56f58a7 100644 --- a/vercel.json +++ b/vercel.json @@ -5,7 +5,15 @@ "headers": [ { "key": "Cache-Control", - "value": "public, s-maxage=21600, max-age=21600, stale-while-revalidate=86400" + "value": "no-store, no-cache, must-revalidate, proxy-revalidate" + }, + { + "key": "Pragma", + "value": "no-cache" + }, + { + "key": "Expires", + "value": "0" } ] } From e2cff3f5bfdb3805640bcf23f41b601d64d9049a Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:18:18 -0800 Subject: [PATCH 37/44] WIP: remove debug-env endpoint --- .github/copilot-instructions.md | 11 ++++------- app/api/debug-env/route.ts | 24 ------------------------ 2 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 app/api/debug-env/route.ts diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7b23b02..f1600f4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -20,7 +20,6 @@ - `/api/acquia/visits` - Fetches visits metrics with pagination - `/api/acquia/views` - Fetches views metrics with pagination - `/api/cache` - Cache management endpoint (GET/DELETE) -- `/api/debug-env` - Environment variable debugging **Core Services** (`lib/`): - `lib/acquia-api.ts` - Acquia Cloud API client with hybrid caching and pagination @@ -185,11 +184,10 @@ 6. Add conditional render in tab content section ### Debugging Acquia API Issues -1. Check environment variables via `/api/debug-env` +1. Check API route error responses for detailed error info including envCheck 2. Verify auth credentials are set correctly (no quotes) -3. Check API route error responses for detailed error info -4. Monitor cache behavior via console logs -5. Use "Clear Cache" functionality to force fresh data +3. Monitor cache behavior via console logs +4. Use "Clear Cache" functionality to force fresh data ### Environment Setup @@ -223,7 +221,6 @@ app/ api/ # API routes (Next.js 15 route handlers) acquia/ # Acquia Cloud API proxy routes cache/ # Cache management endpoint - debug-env/ # Environment debugging applications/ # Applications pages [uuid]/ # Individual application detail page.tsx # Applications overview table @@ -250,7 +247,7 @@ utilities/ # Helper utilities - Use basic auth credentials: sws/sws - Test API endpoints directly via browser DevTools - Check cache behavior via console logs -- Verify environment variables via `/api/debug-env` +- Check API error responses for envCheck debugging info **Production Checklist**: - Verify all environment variables set in Vercel diff --git a/app/api/debug-env/route.ts b/app/api/debug-env/route.ts deleted file mode 100644 index 881b54d..0000000 --- a/app/api/debug-env/route.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { NextRequest, NextResponse } from 'next/server'; - -export async function GET(request: NextRequest) { - return NextResponse.json({ - allEnvVars: { - NODE_ENV: process.env.NODE_ENV, - VERCEL: process.env.VERCEL, - VERCEL_ENV: process.env.VERCEL_ENV, - VERCEL_URL: process.env.VERCEL_URL, - VERCEL_REGION: process.env.VERCEL_REGION, - }, - detectionResults: { - isVercel1: process.env.VERCEL === '1', - isVercelEnv: !!process.env.VERCEL_ENV, - isVercelUrl: !!process.env.VERCEL_URL, - nodeEnvDev: process.env.NODE_ENV === 'development', - nodeEnvProd: process.env.NODE_ENV === 'production', - }, - recommendedDetection: { - isVercel: process.env.VERCEL === '1' || !!process.env.VERCEL_ENV, - isLocal: process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV - } - }); -} \ No newline at end of file From 12c974b19d64728ec41cbc76da79c996eafd7b36 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:20:06 -0800 Subject: [PATCH 38/44] Further cache duration alignment --- lib/cache-hybrid.ts | 2 +- lib/cache.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index ed62323..cd09c0c 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -1,6 +1,6 @@ import { unstable_cache } from 'next/cache'; -// Cache TTL in milliseconds (5 minutes) +// 5-minute cache TTL (consistent across all caching layers) const CACHE_TTL_MS = 5 * 60 * 1000; // Cache buster that gets updated when cache is cleared diff --git a/lib/cache.ts b/lib/cache.ts index 33bf8f9..ab20e1f 100644 --- a/lib/cache.ts +++ b/lib/cache.ts @@ -3,7 +3,7 @@ import path from 'path'; import crypto from 'crypto'; const CACHE_DIR = path.join(process.cwd(), '.cache'); -const CACHE_TTL = 60 * 2 * 1000; // 2 minutes in milliseconds (was 6 hours) +const CACHE_TTL = 5 * 60 * 1000; // 5 minutes in milliseconds (matches hybrid cache system) // Ensure cache directory exists async function ensureCacheDir() { From aa09035ececb05ac02f793d20e7e2e17a5472d1e Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:25:00 -0800 Subject: [PATCH 39/44] Clean up unused variables --- app/applications/page.tsx | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/applications/page.tsx b/app/applications/page.tsx index 86541ca..c5bb755 100644 --- a/app/applications/page.tsx +++ b/app/applications/page.tsx @@ -179,9 +179,6 @@ function getAppStats(apps: any[], views: any[], visits: any[]) { export default function ApplicationsPage() { const [loading, setLoading] = useState(false); const [error, setError] = useState(null); - const [apps, setApps] = useState([]); - const [views, setViews] = useState([]); - const [visits, setVisits] = useState([]); const [stats, setStats] = useState([]); const [elapsedTime, setElapsedTime] = useState(null); @@ -196,9 +193,7 @@ export default function ApplicationsPage() { console.log('📊 Fetching applications data with cache-busting parameter'); const { apps, views, visits } = await fetchData(Date.now().toString()); - const calculatedStats = getAppStats(apps, views, visits); setApps(apps); - setViews(views); - setVisits(visits); + const calculatedStats = getAppStats(apps, views, visits); setStats(calculatedStats); console.log('✅ Applications data loaded successfully'); From 92649bf166f3c1c7c6a01d88746f1c783d0c56ef Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:36:24 -0800 Subject: [PATCH 40/44] Update cache handling on API routes --- app/api/acquia/views/route.ts | 8 +++----- app/api/acquia/visits/route.ts | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/app/api/acquia/views/route.ts b/app/api/acquia/views/route.ts index 469d36f..c1badc9 100644 --- a/app/api/acquia/views/route.ts +++ b/app/api/acquia/views/route.ts @@ -81,11 +81,9 @@ export async function GET(request: NextRequest) { ['views', subscriptionUuid] // Cache tags ); - // Add cache status to response - const response = NextResponse.json({ - ...result, - cached: true - }); + // Return the result directly without cache status metadata + // (hybrid caching is transparent - data is always fresh within 5-minute TTL) + const response = NextResponse.json(result); // Disable browser caching completely - server handles all caching // no-store prevents any caching, no-cache forces revalidation diff --git a/app/api/acquia/visits/route.ts b/app/api/acquia/visits/route.ts index fb67c98..187ef45 100644 --- a/app/api/acquia/visits/route.ts +++ b/app/api/acquia/visits/route.ts @@ -81,11 +81,9 @@ export async function GET(request: NextRequest) { ['visits', subscriptionUuid] // Cache tags ); - // Add cache status to response - const response = NextResponse.json({ - ...result, - cached: true, // This will be overridden by the actual cache status - }); + // Return the result directly without cache status metadata + // (hybrid caching is transparent - data is always fresh within 5-minute TTL) + const response = NextResponse.json(result); // Disable browser caching completely - server handles all caching // no-store prevents any caching, no-cache forces revalidation From f4951968368e8ccdc7d60aa569f4b5237fcdcfa7 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:40:03 -0800 Subject: [PATCH 41/44] fixup! comments --- lib/acquia-api.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/acquia-api.ts b/lib/acquia-api.ts index 52c7940..58d2ab4 100644 --- a/lib/acquia-api.ts +++ b/lib/acquia-api.ts @@ -265,7 +265,8 @@ class AcquiaApiServiceFixed { } async getApplications(): Promise { - // Temporarily disable caching to debug + // Caching is now handled at the API route level via hybrid caching system + // This service layer focuses purely on API communication try { console.log(`🔍 Fetching all applications from API`); const response = await this.makeAuthenticatedRequest('/applications'); @@ -542,7 +543,8 @@ class AcquiaApiServiceFixed { to?: string, resolution?: string ): Promise { - // Temporarily disable caching to debug + // Caching is now handled at the API route level via hybrid caching system + // This method focuses on pagination and data fetching from Acquia API let allData: T[] = []; let currentPage = 1; let totalPages = 1; From 9a3f1c3da811bab1bf50e4f26621d32fdb0749ca Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:47:52 -0800 Subject: [PATCH 42/44] Update cache handling for race condition --- .github/copilot-instructions.md | 1 + docs/caching.md | 16 ++++++++++++++++ lib/cache-hybrid.ts | 22 +++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f1600f4..675c872 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -283,6 +283,7 @@ utilities/ # Helper utilities - **Caching System**: `docs/caching.md` (comprehensive caching guide) - **Acquia API**: https://cloud.acquia.com/api (official docs, requires login) +- **Acquia API Rate Limits**: https://docs.acquia.com/acquia-cloud-platform/developing-cloud-platform-api (rate limiting information) - **Decanter**: https://decanter.stanford.edu (Stanford Design System) - **Next.js 15**: https://nextjs.org/docs (App Router only) diff --git a/docs/caching.md b/docs/caching.md index 19e5680..55a308f 100644 --- a/docs/caching.md +++ b/docs/caching.md @@ -14,6 +14,11 @@ The system provides: - **Automatic cache invalidation** on application deployment - **Manual cache clearing** via API endpoint - **Browser cache prevention** to ensure server-side cache control +- **Race condition mitigation** for expired cache scenarios + +## Why Caching Matters + +Acquia API calls are subject to rate limiting. For current rate limit information, see the [Acquia Cloud Platform API documentation](https://docs.acquia.com/acquia-cloud-platform/developing-cloud-platform-api). The hybrid caching system helps ensure we stay within these limits while providing responsive user experience. ## Cache Architecture @@ -24,6 +29,17 @@ The implementation uses a **deployment-based cache invalidation** approach: 1. **Deployment-only versioning**: Cache keys include deployment ID only, automatically invalidating cache on new deployments 2. **Application-layer timestamp validation**: Cached data includes timestamps that are validated on every request 3. **Manual cache clearing**: Limited to revalidation APIs (cache automatically clears on next deployment) +4. **Race condition handling**: Fresh API results are re-cached to help subsequent concurrent requests + +### Expired Cache Handling + +When cached data expires, the system handles race conditions gracefully: + +1. **First request**: Detects expired cache → Makes API call → Returns data + stores fresh cache entry +2. **Concurrent requests**: May also detect expired cache → Make API calls → Return data + store fresh cache entries +3. **Subsequent requests**: Find fresh cache entries → Return cached data (no API calls) + +This approach avoids complex locking mechanisms while reducing API load after the initial cache expiration. ### Browser Cache Prevention diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index cd09c0c..bbf93ca 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -107,12 +107,28 @@ export async function getCachedApiData( console.log(`⏰ Cache age: ${Math.round((Date.now() - new Date(cachedResult.cachedAt).getTime())/1000)}s`); // For expired cache, we need to force a fresh API call - // Since we can't modify unstable_cache keys dynamically, we'll just call the API directly + // Since we can't modify unstable_cache keys dynamically, we'll call the API directly console.log(`🆕 Making fresh API call due to expired cache`); const freshResult = await apiCall(); - // Note: We can't easily update the existing cache entry, but that's okay - // The next request will get a fresh cache entry with the current timestamp + // Store the fresh result back in a new cache entry to help subsequent requests + // Use a slightly different key to avoid conflicts with the expired entry + const freshCacheKey = `${versionedCacheKey}_fresh_${Date.now()}`; + const freshCachedCall = unstable_cache( + async () => ({ + data: freshResult, + cachedAt: new Date().toISOString(), + cacheKey: freshCacheKey + }), + [freshCacheKey], + { tags: ['acquia-api', ...tags] } + ); + + // Store in cache for future requests (fire and forget) + freshCachedCall().catch(error => { + console.warn('⚠️ Failed to cache fresh result (non-critical):', error); + }); + return freshResult; } From 8a4fcdf9120087a134423546268e4eda7ac79bf4 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:51:02 -0800 Subject: [PATCH 43/44] Move environment detection to runtime --- app/api/cache/route.ts | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/app/api/cache/route.ts b/app/api/cache/route.ts index 04d1565..2364f80 100644 --- a/app/api/cache/route.ts +++ b/app/api/cache/route.ts @@ -1,12 +1,12 @@ import { NextRequest, NextResponse } from 'next/server'; -// Fixed environment detection based on actual Vercel env vars -const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; -const isVercel = !!process.env.VERCEL_ENV; - export async function DELETE(request: NextRequest) { try { - console.log('🔍 Environment detection:', { + // Check environment at runtime, not module load time (matches cache-hybrid.ts pattern) + const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; + const isVercel = !!process.env.VERCEL_ENV; + + console.log('🔍 Environment detection (RUNTIME):', { isLocal, isVercel, NODE_ENV: process.env.NODE_ENV, @@ -38,6 +38,10 @@ export async function DELETE(request: NextRequest) { }); } } catch (error) { + // Environment detection for error reporting (also at runtime) + const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; + const isVercel = !!process.env.VERCEL_ENV; + console.error('❌ Cache management error:', error); return NextResponse.json( { @@ -57,6 +61,10 @@ export async function DELETE(request: NextRequest) { } export async function GET(request: NextRequest) { + // Check environment at runtime for GET endpoint as well + const isLocal = process.env.NODE_ENV === 'development' && !process.env.VERCEL_ENV; + const isVercel = !!process.env.VERCEL_ENV; + return NextResponse.json({ message: 'Cache management API', environment: isLocal ? 'Local (file cache)' : 'Vercel (cache-buster)', From 558f5dcbd49a6720d9cc6de00072e85e5ce8fcd3 Mon Sep 17 00:00:00 2001 From: John Bickar Date: Mon, 24 Nov 2025 15:52:35 -0800 Subject: [PATCH 44/44] remove unused getCacheBuster function --- lib/cache-hybrid.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/cache-hybrid.ts b/lib/cache-hybrid.ts index bbf93ca..7559021 100644 --- a/lib/cache-hybrid.ts +++ b/lib/cache-hybrid.ts @@ -3,10 +3,6 @@ import { unstable_cache } from 'next/cache'; // 5-minute cache TTL (consistent across all caching layers) const CACHE_TTL_MS = 5 * 60 * 1000; -// Cache buster that gets updated when cache is cleared -// This is stored in memory but we also use deployment ID to bust cache on deploy -let cacheBusterTimestamp: number | null = null; - // Get cache version based on deployment ID (changes with each deploy) // This ensures cache is automatically invalidated on new deployments function getCacheVersion(): string { @@ -17,22 +13,6 @@ function getCacheVersion(): string { return deploymentId; } -// Get the current cache buster timestamp -async function getCacheBuster(): Promise { - if (cacheBusterTimestamp === null) { - cacheBusterTimestamp = Date.now(); - } - return cacheBusterTimestamp; -} - -// Update the cache buster to force cache invalidation -export async function updateCacheBuster(): Promise { - const newTimestamp = Date.now(); - cacheBusterTimestamp = newTimestamp; - console.log('🔄 Cache buster updated:', newTimestamp); - return newTimestamp; -} - // Check if cached data is still valid based on timestamp function isCacheDataValid(cachedTimestamp: string): boolean { const now = Date.now();