url cleanup upgrade

.env.development

@@ -4,6 +4,9 @@ BASE_REDIRECT_URL=http://localhost:3000/u/
 LOG_LEVEL=debug
 npm_package_type=module
 
+## create/update
+URL_EXPIRE_FROM=create
+
 ########## Storage ##########
 ## InMemory/Relational
 STORAGE_DRIVER=InMemory

src/config/__test__/helpers.ts

@@ -10,6 +10,7 @@ export function getRawConfig(): RawConfig {
 		url: {
 			matchPattern: 'mock-patter',
 			lifetime: '120',
+			urlExpireFrom: 'create'
 		},
 		storage: {
 			driverName: 'InMemory',

src/config/__test__/normalize.test.ts

@@ -23,10 +23,15 @@ test('Happy flow', () => {
 			lifetimeMs: ms(url.lifetime),
 			matchPattern: url.matchPattern,
 			cleanupIntervalMs: config.url.cleanupIntervalMs, // We're not yet testing this
+			urlExpireFrom: config.url.urlExpireFrom
 		},
 		storage: {
+			appName: appName,
 			driverName: storage.driverName as StorageDriverName,
+			cleanupIntervalMs: config.url.cleanupIntervalMs,
 			driverConfig: storage.driverName === StorageDriverName.Relational ? storage.relationalDriverConfig : {},
+			urlExpireFrom: url.urlExpireFrom,
+			lifetimeMs: ms(url.lifetime),
 		},
 		auth: {
 			driverName: auth.driverName,

src/config/index.ts

@@ -16,6 +16,7 @@ const rawConfig: RawConfig = {
 	url: {
 		matchPattern: process.env.URL_MATCH_PATTERN || '**',
 		lifetime: process.env.URL_LIFETIME || '7 days',
+		urlExpireFrom: process.env.URL_EXPIRE_FROM || 'create',
 	},
 	storage: {
 		driverName: process.env.STORAGE_DRIVER || '',

src/config/normalize.ts

@@ -26,7 +26,6 @@ export function normalizeConfig({
 	const minimumCleanupTime = Math.max(idealCleanupInterval, MIN_URL_CLEANUP_INTERVAL_MS)
 	// No more than the maximum
 	const cleanupIntervalMs = Math.min(minimumCleanupTime, MAX_URL_CLEANUP_INTERVAL_MS)
-
 	return {
 		port,
 		logLevel,
@@ -35,16 +34,23 @@ export function normalizeConfig({
 		baseRedirectUrl,
 		url: {
 			lifetimeMs: ms(url.lifetime),
+			urlExpireFrom: url.urlExpireFrom,
 			matchPattern: url.matchPattern,
 			cleanupIntervalMs,
 		},
 		storage: {
 			driverName: storage.driverName as StorageDriverName,
 			driverConfig: storage.driverName === StorageDriverName.Relational ? storage.relationalDriverConfig : {},
+			urlExpireFrom: url.urlExpireFrom,
+			cleanupIntervalMs: cleanupIntervalMs,
+			lifetimeMs:ms(url.lifetime),
+			appName: appName
 		},
+
 		auth: {
 			driverName: auth.driverName as AuthDriverName,
 			driverConfig: auth.bearerTokenDriverConfig,
 		},
 	}
 }
+

src/config/types.ts

@@ -10,6 +10,7 @@ export interface RawConfig {
 	url: {
 		matchPattern: string
 		lifetime: string
+		urlExpireFrom: string
 	}
 	storage: {
 		driverName: string
@@ -39,9 +40,10 @@ export interface Config {
 	url: {
 		matchPattern: string
 		lifetimeMs: number
+		urlExpireFrom: string
 		cleanupIntervalMs: number
 	}
 	baseRedirectUrl: string
-	storage: Omit<StorageConfig, 'appName' | 'lifetimeMs' | 'cleanupIntervalMs'>
+	storage: StorageConfig
 	auth: AuthConfig
 }

src/config/validate.ts

@@ -11,6 +11,7 @@ export function validateConfig(rawConfig: RawConfig): boolean {
 	validateStorageDriver(rawConfig.storage)
 	validateAuthDriver(rawConfig.auth)
 	validateUrlLifetime(rawConfig.url.lifetime)
+	validateUrlExpireFrom(rawConfig.url.urlExpireFrom)
 	validateLogLevel(rawConfig.logLevel)
 
 	return true
@@ -101,6 +102,13 @@ function validateUrlLifetime(urlLifetime: string): void {
 	}
 }
 
+function validateUrlExpireFrom(urlExpire: string): void {
+	logger.debug(`Start validateUrlExpireFrom with ${urlExpire}`)
+	if (!urlExpire || (urlExpire !== 'create' && urlExpire !== 'update')) {
+		throw new InvalidConfigError(`URL_EXPIRE_FROM specified is invalid (received ${urlExpire}, expected 'create' or 'update')`)
+	}
+}
+
 function validateLogLevel(logLevel: string) {
 	logger.debug(`Start validateLogLevel with ${logLevel}`)
 	const levelValues = Object.keys(logger.levels.values)

src/index.ts

@@ -9,14 +9,8 @@ import { createApp } from './app.js'
 logger.info(`Logger level defined as ${config.logLevel}`)
 logger.setLevel(config.logLevel)
 
-// Storage
-const storage = new Storage({
-	appName: config.appName,
-	driverName: config.storage.driverName,
-	driverConfig: config.storage.driverConfig,
-	lifetimeMs: config.url.lifetimeMs,
-	cleanupIntervalMs: config.url.cleanupIntervalMs,
-})
+
+const storage = new Storage(config.storage)
 await storage.initialize()
 
 // Fastify

src/services/storage/drivers/inMemory/index.ts

@@ -3,8 +3,14 @@ import { NotFoundError , GeneralError } from '../../../../errors/errors.js'
 import { InMemoryStorageConfig } from '../../types/config.js'
 import type { StorageDriver } from '../../types/index.js'
 import type { StoredUrl, UrlWithInformation, UrlRequestData, UrlInformation } from '../../types/url.js'
+import { logger } from '../../../logger/logger.js'
 
 export class InMemoryStorage implements StorageDriver {
+	private urlExpireFrom
+
+	constructor(private config: InMemoryStorageConfig) {
+		this.urlExpireFrom = config.urlExpireFrom
+	}
 	data: { urls: Map<string, StoredUrl>; urlInformation: Map<string, UrlInformation> } = {
 		urls: new Map(),
 		urlInformation: new Map(),
@@ -39,12 +45,15 @@ export class InMemoryStorage implements StorageDriver {
 			this.storage.data.urls.delete(id)
 		}
 		public async deleteOverdue(timespanMs: number): Promise<number> {
+			logger.debug('urlExpireFrom is ' + "'" + this.storage.urlExpireFrom + "'")
 			const deleteBefore = new Date().getTime() - timespanMs
 			let deletedCount = 0
 
 			this.storage.data.urls.forEach((storedUrl) => {
-				const updatedAt = new Date(storedUrl.updatedAt).getTime()
-				if (updatedAt <= deleteBefore) {
+				const relativeDate = new Date(
+					this.storage.urlExpireFrom === 'update' ? storedUrl.updatedAt : storedUrl.createdAt,
+				).getTime()
+				if (relativeDate <= deleteBefore) {
 					this.storage.data.urls.delete(storedUrl.id)
 					deletedCount++
 				}
@@ -106,11 +115,9 @@ export class InMemoryStorage implements StorageDriver {
 	public async initialize(): Promise<void> {
 		return
 	}
+
 	public async shutdown(): Promise<void> {
 		return
 	}
 
-	// eslint-disable-next-line
-	constructor(public config: InMemoryStorageConfig) {
-	}
 }

src/services/storage/drivers/relational/index.ts

@@ -8,14 +8,17 @@ import { fileURLToPath } from 'url'
 import camelcaseKeys from 'camelcase-keys'
 import { snakeCase } from 'snake-case'
 import type { StoredUrl, UrlWithInformation, UrlRequestData, UrlInformation } from '../../types/url.js'
-import { RelationalStorageConfig } from '../../types/config.js'
+import {InMemoryStorageConfig, RelationalStorageConfig} from '../../types/config.js'
+import { logger } from '../../../logger/logger.js'
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
 
 export class RelationalStorage implements StorageDriver {
 	private db: Knex
+	private urlExpireFrom
 
 	constructor(private config: RelationalStorageConfig) {
+		this.urlExpireFrom = config.urlExpireFrom
 		this.db = Knex({
 			...config.driverConfig,
 			migrations: {
@@ -94,12 +97,23 @@ export class RelationalStorage implements StorageDriver {
 		//https://stackoverflow.com/questions/53859207/deleting-data-from-associated-tables-using-knex-js
 		public async delete(id: string): Promise<void> {
 			await this.storage.db.table<StoredUrl>('urls').where('id', id).delete()
-			return
 		}
 
 		public async deleteOverdue(timespanMs: number): Promise<number> {
-			const deleteBefore = new Date(new Date().getTime() - timespanMs)
-			return await this.storage.db.table<StoredUrl>('urls').where('updatedAt', '<', deleteBefore).delete()
+			logger.debug('urlExpireFrom is ' + "'" + this.storage.urlExpireFrom + "'")
+			const deleteBefore = new Date().getTime() - timespanMs
+			let deletedCount = 0
+			const urls: StoredUrl[] = await this.storage.db.table<StoredUrl>('urls')
+			urls.forEach((value) => {
+				const relativeDate = new Date(
+					this.storage.urlExpireFrom === 'update' ? value.updatedAt : value.createdAt,
+				).getTime()
+				if (relativeDate <= deleteBefore) {
+					this.delete(value.id)
+					deletedCount++
+				}
+			})
+			return deletedCount
 		}
 
 		public async edit(id: string, url: string): Promise<StoredUrl> {

src/services/storage/drivers/relational/types.ts

@@ -1,4 +1,3 @@
 import type Knex from 'knex'
 
-// eslint-disable-next-line @typescript-eslint/no-empty-interface
-export interface RelationalStorageDriverConfig extends Knex.Config {}
+export type RelationalStorageDriverConfig = Knex.Config

src/services/storage/index.ts

@@ -23,6 +23,7 @@ export class Storage implements StorageDriver {
 				throw new InvalidConfigError(`Invalid url storage driver selected.`)
 		}
 	}
+
 	get config(): StorageConfig {
 		return this._config
 	}

src/services/storage/types/config.ts

@@ -10,6 +10,7 @@ export interface BaseConfig {
 	appName: string
 	lifetimeMs: number
 	cleanupIntervalMs: number
+	urlExpireFrom: string
 }
 
 export interface RelationalStorageConfig extends BaseConfig {