Skip to content

Fix excessive Redis log spam when Redis is unavailable #649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
valentin0h wants to merge 2 commits into
base: dev
Choose a base branch
from
Draft

Fix excessive Redis log spam when Redis is unavailable #649

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f4cd2ad
Fix excessive Redis log spam when Redis is unavailable
valentin0h Feb 21, 2026
36c1471
Remove reconnect retry cap so Redis can recover after outage
valentin0h Feb 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 28 additions & 14 deletions services/cache/kv.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ const REDIS_URL = process.env.REDIS_URL;
// Create Redis client only if REDIS_URL is provided
let redis: ReturnType<typeof createClient> | null = null;

/** Tracks whether Redis is currently reachable — used to suppress per-request error spam. */
let redisAvailable = false;

if (REDIS_URL) {
redis = createClient({
url: REDIS_URL,
Expand All @@ -18,24 +21,37 @@ if (REDIS_URL) {
connectTimeout: 10_000,
keepAlive: false, // TCP keepalive
reconnectStrategy: (retries) => {
if (retries > 3) {
return false;
}

const jitter = Math.floor(Math.random() * 100);

return Math.min(retries * 500, 5_000) + jitter;
// Exponential backoff capped at 30s so we can recover when Redis comes back.
// The redisAvailable guard ensures zero per-request overhead while disconnected.
const jitter = Math.floor(Math.random() * 200);
return Math.min(retries * 500, 30_000) + jitter;
},
},
});

redis.on('error', (err) => {
logger.error('Redis Client Error', err);
// Log state transitions once instead of every error event
redis.on('error', () => {
if (redisAvailable) {
redisAvailable = false;
logger.warn(
'Redis connection lost — cache operations will fall through to DB',
);
}
});

redis.on('ready', () => {
if (!redisAvailable) {
redisAvailable = true;
logger.info('Redis connected');
}
});

// Connect to Redis
if (!redis.isOpen) {
redis.connect().catch(console.error);
redis.connect().catch(() => {
// Intentionally swallowed — the 'error' event handler above already
// logs the transition to unavailable state once.
});
}
}

Expand Down Expand Up @@ -205,7 +221,7 @@ export const invalidateMultiple = async ({
};

export const get = async (key: string) => {
if (!redis) {
if (!redis || !redisAvailable) {
return null;
}

Expand All @@ -218,7 +234,6 @@ export const get = async (key: string) => {

return null;
} catch (e) {
logger.error('CACHE: error getting from Redis', { error: e });
cacheMetrics.recordError('get');

return null;
Expand All @@ -228,7 +243,7 @@ export const get = async (key: string) => {
// const DEFAULT_TTL = 3600 * 24 * 30; // 3600 * 24 = 1 day
const DEFAULT_TTL = 3600; // short TTL for testing
export const set = async (key: string, data: unknown, ttl?: number) => {
if (!redis) {
if (!redis || !redisAvailable) {
return;
}

Expand All @@ -240,7 +255,6 @@ export const set = async (key: string, data: unknown, ttl?: number) => {
await redis.setEx(key, ttl || DEFAULT_TTL, serializedData);
}
} catch (e) {
logger.error('CACHE: error setting to Redis', { error: e });
cacheMetrics.recordError('set');
}
};