From 3cad7343fc688bbe00e233407514ff03619a88c0 Mon Sep 17 00:00:00 2001 From: xkd <6140f9@gmail.com> Date: Thu, 2 Oct 2025 19:50:58 +0300 Subject: [PATCH 1/3] Add support for custom retry strategies --- docs/http_block.md | 29 ++++++++++ lib/httpBlock.ts | 3 +- lib/index.ts | 5 +- lib/request.ts | 42 +++++++-------- lib/retryStrategy.ts | 59 +++++++++++++++++++++ tests/request.test.ts | 119 +++++++++++++++++++++++++++++++++++++++++- 6 files changed, 232 insertions(+), 25 deletions(-) create mode 100644 lib/retryStrategy.ts diff --git a/docs/http_block.md b/docs/http_block.md index a9de787..1ccd583 100644 --- a/docs/http_block.md +++ b/docs/http_block.md @@ -435,6 +435,35 @@ block: { }, ``` +### Свои стратегии ретраев + +Позволяет полностью контролировать процесс ретрая запросов. Это необходимо для того, чтобы можно было реализовать любые кастомные стратегии перезапросов, например, более сложные алгоритмы для борьбы с [retry storm](https://learn.microsoft.com/en-us/azure/architecture/antipatterns/retry-storm/) + +```js +block: { + getRetryStragety: ( { requestOptions, request } ) => { + return new MyAwesomeRetryStrategy({ request, ... }); + }, +}, +``` + +В качестве аргумента приходят `requestOptions`. Это необходимо, например, для того, чтобы вы имели полную информацию о запросе при построении ключей во внешнем хранилище. Также приходит `request`, с помощью которого можно вызвать новый запрос в случае необходимости. + +Интерфейс стратегии должен реализовать следующие методы: + +```js +export interface DescriptRetryStrategyInterface { + // Метод, который будет оборачивать оригинальный запрос + makeRequest: () => Promise; +} +``` + +Таким образом, с помощью getRetryStrategy вы можете передать управление выполнением запроса. + +### Важный момент: + +Поддержка и синхронизация других параметров, связанных с ретраем, остаются на вашей стороне. Это связано с тем, что некоторые опции могут не подходить напрямую: например, `retryTimeout` может вычисляться динамически внутри стратегии. + ## `prepareRequestOptions` diff --git a/lib/httpBlock.ts b/lib/httpBlock.ts index 9ef28a6..f1c0bc6 100644 --- a/lib/httpBlock.ts +++ b/lib/httpBlock.ts @@ -79,7 +79,7 @@ export interface DescriptHttpBlockDescription< HTTPResult, > extends Pick< DescriptRequestOptions, - 'isError' | 'isRetryAllowed' | 'retryTimeout' + 'isError' | 'isRetryAllowed' | 'retryTimeout' | 'getRetryStrategy' > { // sync with EVALUABLE_PROPS agent?: DescriptHttpBlockDescriptionCallback; @@ -289,6 +289,7 @@ class HttpBlock< isError: block.isError, isRetryAllowed: block.isRetryAllowed, retryTimeout: block.retryTimeout, + getRetryStrategy: block.getRetryStrategy, body: null, ...( EVALUABLE_PROPS.reduce((ret, prop) => { diff --git a/lib/index.ts b/lib/index.ts index c17ec09..b011f78 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -6,7 +6,7 @@ import Logger, { EVENT } from './logger'; import type { LoggerEvent, LoggerInterface } from './logger'; import Cache, { CacheInterface } from './cache'; -import request, { type RequestOptions } from './request'; +import request, { type RequestOptions, type GetRetryStrategyParams } from './request'; import type { GenerateId, DescriptBlockDeps, DescriptBlockId } from './depsDomain'; import Block from './block'; import ArrayBlock from './arrayBlock'; @@ -15,6 +15,7 @@ import type { FunctionBlockDefinition } from './functionBlock'; import FunctionBlock from './functionBlock'; import HttpBlock from './httpBlock'; import FirstBlock from './firstBlock'; +import type { RetryStrategyInterface } from './retryStrategy'; import type { DescriptHttpBlockResult, @@ -202,6 +203,8 @@ export { DescriptHttpBlockDescription, DescriptHttpBlockQuery, DescriptHttpBlockQueryValue, + RetryStrategyInterface, + GetRetryStrategyParams, GenerateId, DescriptBlockId, InferResultFromBlock, diff --git a/lib/request.ts b/lib/request.ts index 261f115..a4362d8 100644 --- a/lib/request.ts +++ b/lib/request.ts @@ -20,13 +20,14 @@ import { EVENT } from './logger'; import type { Deffered } from './getDeferred'; import getDeferred from './getDeferred'; import type Cancel from './cancel'; -import type { DescriptError, Reason } from './error'; +import { DescriptError, Reason } from './error'; import { createError, ERROR_ID } from './error'; import is_plain_object from './isPlainObject'; import extend from './extend'; import type http from 'node:http'; import type { DescriptHttpResult, DescriptJSON } from './types'; +import { BaseRetryStrategy, RetryStrategyInterface, RetryStrategyRequest } from './retryStrategy'; // --------------------------------------------------------------------------------------------------------------- // @@ -80,6 +81,17 @@ export interface DescriptRequestOptions { bodyCompress?: ZlibOptions; agent?: HttpsAgent | HttpsAgentOptions | false | null; + + getRetryStrategy?: ({ + requestOptions, + logger, + }: GetRetryStrategyParams) => RetryStrategyInterface; +} + +export interface GetRetryStrategyParams { + requestOptions: RequestOptions; + logger: LoggerInterface; + request: RetryStrategyRequest; } export interface BlockRequestOptions { @@ -577,42 +589,28 @@ class DescriptRequest { async function request(options: DescriptRequestOptions, logger: LoggerInterface, cancel: Cancel): Promise { const requestOptions = new RequestOptions(options); - while (true) { + const request = async() => { const req = new DescriptRequest(requestOptions, logger, cancel); try { - const result = await req.start(); - - return result; - + return await req.start(); } catch (error) { if (error.error.statusCode === 429 || error.error.statusCode >= 500) { // Удаляем сокет, чтобы не залипать на отвечающем ошибкой бекэнде. req.destroyRequestSocket(); } - if (requestOptions.retries < requestOptions.maxRetries && requestOptions.isRetryAllowed?.(error, requestOptions)) { - requestOptions.retries++; + throw error; + } + }; - if (requestOptions.retryTimeout > 0) { - await waitFor(requestOptions.retryTimeout); - } + const retryStrategy = options.getRetryStrategy?.({ requestOptions, logger, request }) || new BaseRetryStrategy({ requestOptions, request }); - } else { - throw error; - } - } - } + return await retryStrategy.makeRequest(); } request.DEFAULT_OPTIONS = DEFAULT_OPTIONS; -function waitFor(timeout: number) { - return new Promise((resolve) => { - setTimeout(resolve, timeout); - }); -} - class ZstdDecompress extends Transform { receivedLength: number; receivedChunks: Array; diff --git a/lib/retryStrategy.ts b/lib/retryStrategy.ts new file mode 100644 index 0000000..aa26173 --- /dev/null +++ b/lib/retryStrategy.ts @@ -0,0 +1,59 @@ +import { DescriptError } from './error'; +import { RequestOptions } from './request'; +import type { DescriptHttpResult } from './types'; + +export type RetryStrategyRequest = () => Promise; + +export interface RetryStrategyInterface { + makeRequest: () => Promise; +} + +function waitFor(timeout: number) { + return new Promise((resolve) => { + setTimeout(resolve, timeout); + }); +} + +interface BaseRetryStrategyConstructorParams { + requestOptions: RequestOptions; + request: RetryStrategyRequest; +} + +export class BaseRetryStrategy implements RetryStrategyInterface { + requestOptions: RequestOptions; + request: RetryStrategyRequest; + + constructor({ requestOptions, request }: BaseRetryStrategyConstructorParams) { + this.request = request; + this.requestOptions = requestOptions; + } + + private async retry() { + this.requestOptions.retries++; + + if (this.requestOptions.retryTimeout > 0) { + await waitFor(this.requestOptions.retryTimeout); + } + + return this.makeRequest(); + } + + private isRetryAllowed(error: DescriptError) { + return ( + this.requestOptions.retries < this.requestOptions.maxRetries && + this.requestOptions.isRetryAllowed?.(error, this.requestOptions) + ); + } + + async makeRequest(): Promise { + try { + return await this.request(); + } catch (error) { + if (this.isRetryAllowed(error)) { + return this.retry(); + } else { + throw error; + } + } + } +} diff --git a/tests/request.test.ts b/tests/request.test.ts index 58b6a9f..96e7780 100644 --- a/tests/request.test.ts +++ b/tests/request.test.ts @@ -16,7 +16,8 @@ import https_ from 'https'; import http from 'http'; import fs_ from 'fs'; import path_ from 'path'; -import type { Cancel, LoggerInterface } from '../lib'; +import type { Cancel, LoggerInterface, RetryStrategyInterface, GetRetryStrategyParams } from '../lib'; +import { DescriptHttpResult } from 'lib/types'; // --------------------------------------------------------------------------------------------------------------- // @@ -1093,6 +1094,122 @@ describe('request', () => { }); + describe('getRetryStrategy', () => { + const PORT = 9000; + const CUSTOM_RETRY_ERROR_ID = 'CUSTOM_RETRY_ERROR'; + + const doRequest = getDoRequest({ + protocol: 'http:', + hostname: '127.0.0.1', + port: PORT, + pathname: '/', + }); + + const fake = new Server({ + module: http, + listen_options: { + port: PORT, + }, + }); + + beforeAll(() => Promise.all([ + fake.start(), + ])); + + afterAll(() => Promise.all([ + fake.stop(), + ])); + + class MyRetryStrategy implements RetryStrategyInterface { + retries: number; + maxRetries: number; + request: GetRetryStrategyParams['request']; + + constructor(request: GetRetryStrategyParams['request'], maxRetries: number) { + this.retries = 0; + this.request = request; + this.maxRetries = maxRetries; + } + + public async makeRequest(): Promise { + try { + return await this.request(); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return this.retry(); + } + } + + private retry() { + this.retries++; + + if (this.retries < this.maxRetries) { + return this.makeRequest(); + } else { + throw de.error({ + id: CUSTOM_RETRY_ERROR_ID, + }); + } + } + } + + it('The custom retry strategy is called as expected', async() => { + const path = getPath(); + const statusCode = 404; + const content = 'Hello!'; + + fake.add(path, [ + { + statusCode: statusCode, + }, + { + statusCode: statusCode, + }, + { + statusCode: 200, + content: content, + }, + ]); + + let makeRequestSpy; + + const result = await doRequest({ + pathname: path, + getRetryStrategy: ({ request }) => { + const instance = new MyRetryStrategy(request, 5); + makeRequestSpy = vi.spyOn(instance, 'makeRequest'); + return instance; + }, + }); + + expect(makeRequestSpy).toHaveBeenCalledTimes(3); + expect(result.statusCode).toBe(200); + expect(result.body?.toString()).toBe(content); + }); + + it('The custom retry strategy throws its error', async() => { + const path = getPath(); + + fake.add(path, [ + { + statusCode: 404, + }, + ]); + + try { + await doRequest({ + pathname: path, + getRetryStrategy: ({ request }) => { + const instance = new MyRetryStrategy(request, 1); + return instance; + }, + }); + } catch ({ error }) { + expect(error.id).toBe(CUSTOM_RETRY_ERROR_ID); + } + }); + }); + describe('aborted request', () => { describe('no bytes sent', () => { From 10cfede4830ccd2849c9772f6f6cc142de85b9e2 Mon Sep 17 00:00:00 2001 From: xkd <6140f9@gmail.com> Date: Fri, 3 Oct 2025 09:57:24 +0300 Subject: [PATCH 2/3] Bump version to 4.0.19 --- CHANGELOG.md | 4 ++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9803ad..9a19af9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## `4.0.19` + + * Добавлена возможность использования пользовательских стратегий ретраев. + ## `4.0.18` * Правки типов. after может корректно возвращать undefined diff --git a/package-lock.json b/package-lock.json index 8240cda..b4385d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "descript", - "version": "4.0.18", + "version": "4.0.19", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "descript", - "version": "4.0.18", + "version": "4.0.19", "license": "MIT", "dependencies": { "@fengkx/zstd-napi": "^0.1.0" diff --git a/package.json b/package.json index 8d371fd..67d0093 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ }, "name": "descript", "description": "descript", - "version": "4.0.18", + "version": "4.0.19", "homepage": "https://github.com/descript-org/descript", "repository": { "type": "git", From 7bb457f20eb2c12cc76c890e249611b6307e88a1 Mon Sep 17 00:00:00 2001 From: xkd <6140f9@gmail.com> Date: Fri, 3 Oct 2025 10:57:53 +0300 Subject: [PATCH 3/3] Fix test --- tests/request.test.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/request.test.ts b/tests/request.test.ts index 96e7780..885bde5 100644 --- a/tests/request.test.ts +++ b/tests/request.test.ts @@ -1171,18 +1171,21 @@ describe('request', () => { }, ]); - let makeRequestSpy; + let requestMock: any; const result = await doRequest({ pathname: path, getRetryStrategy: ({ request }) => { - const instance = new MyRetryStrategy(request, 5); - makeRequestSpy = vi.spyOn(instance, 'makeRequest'); - return instance; + if (!requestMock) { + requestMock = vi.fn(request); + request = requestMock; + }; + + return new MyRetryStrategy(request, 5); }, }); - expect(makeRequestSpy).toHaveBeenCalledTimes(3); + expect(requestMock).toHaveBeenCalledTimes(3); expect(result.statusCode).toBe(200); expect(result.body?.toString()).toBe(content); });