o11y with context-aware logging and distributed tracing

dd-trace.js

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2025 GoodRx, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const tracer = require('dd-trace').init({
+  serviceMapping: {
+    redis: 'lifecycle-redis',
+    ioredis: 'lifecycle-redis',
+    pg: 'lifecycle-postgres',
+  },
+});
+
+const blocklist = [/^\/api\/health/, /^\/api\/jobs/, /^\/_next\/static/, /^\/_next\/webpack-hmr/];
+
+tracer.use('http', {
+  server: {
+    blocklist,
+  },
+  client: {
+    blocklist,
+  },
+});
+
+tracer.use('next', {
+  blocklist,
+});
+
+tracer.use('net', false);
+tracer.use('dns', false);

helm/environments/local/lifecycle.yaml

@@ -66,6 +66,8 @@ components:
           value: web
         - name: PORT
           value: '80'
+        - name: DD_TRACE_ENABLED
+          value: 'false'
       ports:
         - name: http
           containerPort: 80
@@ -122,6 +124,8 @@ components:
           value: '10000'
         - name: LIFECYCLE_UI_HOSTHAME_WITH_SCHEME
           value: 'http://localhost:8000'
+        - name: DD_TRACE_ENABLED
+          value: 'false'
       ports:
         - name: http
           containerPort: 80

next-env.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
+/// <reference types="next/navigation-types/compat/navigation" />
 
 // NOTE: This file should not be edited
-// see https://nextjs.org/docs/basic-features/typescript for more information.
+// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information.

package.json

@@ -7,17 +7,17 @@
   ],
   "scripts": {
     "babel-node": "babel-node --extensions '.ts'",
-    "dev": "LOG_LEVEL=debug ts-node -r tsconfig-paths/register --project tsconfig.server.json ws-server.ts | pino-pretty -c -t HH:MM -i pid,hostname,filename -o '{msg}'",
+    "dev": "LOG_LEVEL=debug ts-node -r ./dd-trace.js -r tsconfig-paths/register --project tsconfig.server.json ws-server.ts | pino-pretty -c -t HH:MM -i pid,hostname,filename -o '{msg}'",
     "build": "next build && tsc --project tsconfig.server.json && tsc-alias -p tsconfig.server.json",
-    "start": "NEXT_MANUAL_SIG_HANDLE=true NODE_ENV=production node .next/ws-server.js",
+    "start": "NEXT_MANUAL_SIG_HANDLE=true NODE_ENV=production node -r ./dd-trace.js .next/ws-server.js",
     "run-prod": "port=5001 pnpm run start",
     "knex": "pnpm run knex",
     "test": "NODE_ENV=test jest --maxWorkers=75%",
     "lint": "eslint --ext .ts src",
     "lint:fix": "pnpm run lint --fix",
     "ts-check": "tsc --project tsconfig.json",
-    "db:migrate": "NODE_OPTIONS='--loader ts-node/esm' knex migrate:latest",
-    "db:rollback": "NODE_OPTIONS='--loader ts-node/esm' knex migrate:rollback",
+    "db:migrate": "tsx node_modules/knex/bin/cli.js migrate:latest",
+    "db:rollback": "tsx node_modules/knex/bin/cli.js migrate:rollback",
     "db:seed": "knex seed:run",
     "prepare": "husky install",
     "generate:jsonschemas": "tsx ./scripts/generateSchemas.ts generatejson",

src/app/api/v2/builds/[uuid]/services/[name]/build-jobs/[jobName]/route.ts

@@ -15,16 +15,12 @@
  */
 
 import { NextRequest } from 'next/server';
-import rootLogger from 'server/lib/logger';
+import { getLogger } from 'server/lib/logger';
 import { LogStreamingService } from 'server/services/logStreaming';
 import { HttpError } from '@kubernetes/client-node';
 import { createApiHandler } from 'server/lib/createApiHandler';
 import { errorResponse, successResponse } from 'server/lib/response';
 
-const logger = rootLogger.child({
-  filename: __filename,
-});
-
 interface RouteParams {
   uuid: string;
   name: string;
@@ -96,7 +92,7 @@ const getHandler = async (req: NextRequest, { params }: { params: RouteParams })
   const { uuid, name: serviceName, jobName } = params;
 
   if (!uuid || !jobName || !serviceName) {
-    logger.warn({ uuid, serviceName, jobName }, 'Missing or invalid path parameters');
+    getLogger().warn(`API: invalid params uuid=${uuid} serviceName=${serviceName} jobName=${jobName}`);
     return errorResponse('Missing or invalid parameters', { status: 400 }, req);
   }
 
@@ -107,7 +103,7 @@ const getHandler = async (req: NextRequest, { params }: { params: RouteParams })
 
     return successResponse(response, { status: 200 }, req);
   } catch (error: any) {
-    logger.error({ err: error, uuid, serviceName, jobName }, 'Error getting log streaming info');
+    getLogger().error({ error }, `API: log streaming info failed jobName=${jobName} service=${serviceName}`);
 
     if (error.message === 'Build not found') {
       return errorResponse('Build not found', { status: 404 }, req);

src/app/api/v2/builds/[uuid]/services/[name]/build-jobs/route.ts

@@ -15,16 +15,12 @@
  */
 
 import { NextRequest } from 'next/server';
-import rootLogger from 'server/lib/logger';
+import { getLogger } from 'server/lib/logger';
 import { HttpError } from '@kubernetes/client-node';
 import { createApiHandler } from 'server/lib/createApiHandler';
 import { errorResponse, successResponse } from 'server/lib/response';
 import { getNativeBuildJobs } from 'server/lib/kubernetes/getNativeBuildJobs';
 
-const logger = rootLogger.child({
-  filename: __filename,
-});
-
 /**
  * @openapi
  * /api/v2/builds/{uuid}/services/{name}/builds:
@@ -87,7 +83,7 @@ const getHandler = async (req: NextRequest, { params }: { params: { uuid: string
   const { uuid, name } = params;
 
   if (!uuid || !name) {
-    logger.warn({ uuid, name }, 'Missing or invalid path parameters');
+    getLogger().warn(`API: invalid params uuid=${uuid} name=${name}`);
     return errorResponse('Missing or invalid uuid or name parameters', { status: 400 }, req);
   }
 
@@ -99,7 +95,7 @@ const getHandler = async (req: NextRequest, { params }: { params: { uuid: string
 
     return successResponse(response, { status: 200 }, req);
   } catch (error) {
-    logger.error({ err: error }, `Error getting build logs for service ${name} in environment ${uuid}.`);
+    getLogger().error({ error }, `API: build logs fetch failed service=${name}`);
 
     if (error instanceof HttpError) {
       if (error.response?.statusCode === 404) {

src/app/api/v2/builds/[uuid]/services/[name]/deploy-jobs/[jobName]/route.ts

@@ -15,16 +15,12 @@
  */
 
 import { NextRequest } from 'next/server';
-import rootLogger from 'server/lib/logger';
+import { getLogger } from 'server/lib/logger';
 import { LogStreamingService } from 'server/services/logStreaming';
 import { HttpError } from '@kubernetes/client-node';
 import { createApiHandler } from 'server/lib/createApiHandler';
 import { errorResponse, successResponse } from 'server/lib/response';
 
-const logger = rootLogger.child({
-  filename: __filename,
-});
-
 interface RouteParams {
   uuid: string;
   name: string;
@@ -96,7 +92,7 @@ const getHandler = async (req: NextRequest, { params }: { params: RouteParams })
   const { uuid, name: serviceName, jobName } = params;
 
   if (!uuid || !jobName || !serviceName) {
-    logger.warn({ uuid, serviceName, jobName }, 'Missing or invalid path parameters');
+    getLogger().warn(`API: invalid params uuid=${uuid} serviceName=${serviceName} jobName=${jobName}`);
     return errorResponse('Missing or invalid parameters', { status: 400 }, req);
   }
 
@@ -107,7 +103,7 @@ const getHandler = async (req: NextRequest, { params }: { params: RouteParams })
 
     return successResponse(response, { status: 200 }, req);
   } catch (error: any) {
-    logger.error({ err: error, uuid, serviceName, jobName }, 'Error getting log streaming info');
+    getLogger().error({ error }, `API: log streaming info failed jobName=${jobName} service=${serviceName}`);
 
     if (error.message === 'Deploy not found') {
       return errorResponse('Deploy not found', { status: 404 }, req);

src/app/api/v2/builds/[uuid]/services/[name]/deploy-jobs/route.ts

@@ -15,16 +15,12 @@
  */
 
 import { NextRequest } from 'next/server';
-import rootLogger from 'server/lib/logger';
+import { getLogger } from 'server/lib/logger';
 import { HttpError } from '@kubernetes/client-node';
 import { createApiHandler } from 'server/lib/createApiHandler';
 import { errorResponse, successResponse } from 'server/lib/response';
 import { getDeploymentJobs } from 'server/lib/kubernetes/getDeploymentJobs';
 
-const logger = rootLogger.child({
-  filename: __filename,
-});
-
 /**
  * @openapi
  * /api/v2/builds/{uuid}/services/{name}/deploys:
@@ -85,7 +81,7 @@ const getHandler = async (req: NextRequest, { params }: { params: { uuid: string
   const { uuid, name } = params;
 
   if (!uuid || !name) {
-    logger.warn({ uuid, name }, 'Missing or invalid path parameters');
+    getLogger().warn(`API: invalid params uuid=${uuid} name=${name}`);
     return errorResponse('Missing or invalid uuid or name parameters', { status: 400 }, req);
   }
 
@@ -97,7 +93,7 @@ const getHandler = async (req: NextRequest, { params }: { params: { uuid: string
 
     return successResponse(response, { status: 200 }, req);
   } catch (error) {
-    logger.error({ err: error }, `Error getting deploy logs for service ${name} in environment ${uuid}.`);
+    getLogger().error({ error }, `API: deploy logs fetch failed service=${name}`);
 
     if (error instanceof HttpError) {
       if (error.response?.statusCode === 404) {

src/pages/api/health.ts

@@ -16,7 +16,7 @@
 
 import { NextApiRequest, NextApiResponse } from 'next';
 import { defaultDb } from 'server/lib/dependencies';
-import logger from 'server/lib/logger';
+import { getLogger } from 'server/lib/logger';
 import RedisClient from 'server/lib/redisClient';
 
 export default async function healthHandler(req: NextApiRequest, res: NextApiResponse) {
@@ -30,7 +30,7 @@ export default async function healthHandler(req: NextApiRequest, res: NextApiRes
     await defaultDb.knex.raw('SELECT 1');
     res.status(200).json({ status: 'Healthy' });
   } catch (error) {
-    logger.error(`Health check failed. Error:\n ${error}`);
+    getLogger().error({ error }, 'Health: check failed');
     return res.status(500).json({ status: 'Unhealthy', error: `An error occurred while performing health check.` });
   }
 }

src/pages/api/v1/admin/ttl/cleanup.ts

@@ -15,14 +15,11 @@
  */
 
 import { NextApiRequest, NextApiResponse } from 'next';
-import rootLogger from 'server/lib/logger';
+import { nanoid } from 'nanoid';
+import { withLogContext, getLogger, LogStage } from 'server/lib/logger';
 import GlobalConfigService from 'server/services/globalConfig';
 import TTLCleanupService from 'server/services/ttlCleanup';
 
-const logger = rootLogger.child({
-  filename: 'v1/admin/ttl/cleanup.ts',
-});
-
 /**
  * @openapi
  * /api/v1/admin/ttl/cleanup:
@@ -160,7 +157,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => {
         return res.status(405).json({ error: `${req.method} is not allowed.` });
     }
   } catch (error) {
-    logger.error(`Error occurred on TTL cleanup operation: \n ${error}`);
+    getLogger().error({ error }, 'TTL: cleanup operation failed');
     res.status(500).json({ error: 'An unexpected error occurred.' });
   }
 };
@@ -172,39 +169,45 @@ async function getTTLConfig(res: NextApiResponse) {
     const ttlConfig = globalConfig.ttl_cleanup;
 
     if (!ttlConfig) {
-      logger.warn('[API] TTL cleanup configuration not found in global config');
+      getLogger().warn('TTL: config not found');
       return res.status(404).json({ error: 'TTL cleanup configuration not found' });
     }
 
     return res.status(200).json({ config: ttlConfig });
   } catch (error) {
-    logger.error(`[API] Error occurred retrieving TTL cleanup config: \n ${error}`);
+    getLogger().error({ error }, 'TTL: config retrieval failed');
     return res.status(500).json({ error: 'Unable to retrieve TTL cleanup configuration' });
   }
 }
 
 async function triggerTTLCleanup(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { dryRun = false } = req.body || {};
+  const correlationId = `api-ttl-cleanup-${Date.now()}-${nanoid(8)}`;
 
-    // Validate dryRun parameter type
-    if (typeof dryRun !== 'boolean') {
-      return res.status(400).json({ error: 'dryRun must be a boolean value' });
-    }
+  return withLogContext({ correlationId }, async () => {
+    try {
+      const { dryRun = false } = req.body || {};
 
-    // Create new service instance and add job to queue
-    const ttlCleanupService = new TTLCleanupService();
-    const job = await ttlCleanupService.ttlCleanupQueue.add('manual-ttl-cleanup', { dryRun });
+      // Validate dryRun parameter type
+      if (typeof dryRun !== 'boolean') {
+        return res.status(400).json({ error: 'dryRun must be a boolean value' });
+      }
 
-    logger.info(`[API] TTL cleanup job triggered manually (job ID: ${job.id}, dryRun: ${dryRun})`);
+      // Create new service instance and add job to queue
+      const ttlCleanupService = new TTLCleanupService();
+      const job = await ttlCleanupService.ttlCleanupQueue.add('manual-ttl-cleanup', { dryRun, correlationId });
 
-    return res.status(200).json({
-      message: 'TTL cleanup job triggered successfully',
-      jobId: job.id,
-      dryRun,
-    });
-  } catch (error) {
-    logger.error(`[API] Error occurred triggering TTL cleanup: \n ${error}`);
-    return res.status(500).json({ error: 'Unable to trigger TTL cleanup job' });
-  }
+      getLogger({ stage: LogStage.CLEANUP_STARTING }).info(
+        `TTL: cleanup job triggered manually jobId=${job.id} dryRun=${dryRun}`
+      );
+
+      return res.status(200).json({
+        message: 'TTL cleanup job triggered successfully',
+        jobId: job.id,
+        dryRun,
+      });
+    } catch (error) {
+      getLogger({ stage: LogStage.CLEANUP_FAILED }).error({ error }, 'TTL: cleanup trigger failed');
+      return res.status(500).json({ error: 'Unable to trigger TTL cleanup job' });
+    }
+  });
 }