diff --git a/CHANGELOG.md b/CHANGELOG.md index e9803ad..9a19af9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## `4.0.19` + + * Добавлена возможность использования пользовательских стратегий ретраев. + ## `4.0.18` * Правки типов. after может корректно возвращать undefined diff --git a/docs/http_block.md b/docs/http_block.md index a9de787..1ccd583 100644 --- a/docs/http_block.md +++ b/docs/http_block.md @@ -435,6 +435,35 @@ block: { }, ``` +### Свои стратегии ретраев + +Позволяет полностью контролировать процесс ретрая запросов. Это необходимо для того, чтобы можно было реализовать любые кастомные стратегии перезапросов, например, более сложные алгоритмы для борьбы с [retry storm](https://learn.microsoft.com/en-us/azure/architecture/antipatterns/retry-storm/) + +```js +block: { + getRetryStragety: ( { requestOptions, request } ) => { + return new MyAwesomeRetryStrategy({ request, ... }); + }, +}, +``` + +В качестве аргумента приходят `requestOptions`. Это необходимо, например, для того, чтобы вы имели полную информацию о запросе при построении ключей во внешнем хранилище. Также приходит `request`, с помощью которого можно вызвать новый запрос в случае необходимости. + +Интерфейс стратегии должен реализовать следующие методы: + +```js +export interface DescriptRetryStrategyInterface { + // Метод, который будет оборачивать оригинальный запрос + makeRequest: () => Promise; +} +``` + +Таким образом, с помощью getRetryStrategy вы можете передать управление выполнением запроса. + +### Важный момент: + +Поддержка и синхронизация других параметров, связанных с ретраем, остаются на вашей стороне. Это связано с тем, что некоторые опции могут не подходить напрямую: например, `retryTimeout` может вычисляться динамически внутри стратегии. + ## `prepareRequestOptions` diff --git a/lib/httpBlock.ts b/lib/httpBlock.ts index 9ef28a6..f1c0bc6 100644 --- a/lib/httpBlock.ts +++ b/lib/httpBlock.ts @@ -79,7 +79,7 @@ export interface DescriptHttpBlockDescription< HTTPResult, > extends Pick< DescriptRequestOptions, - 'isError' | 'isRetryAllowed' | 'retryTimeout' + 'isError' | 'isRetryAllowed' | 'retryTimeout' | 'getRetryStrategy' > { // sync with EVALUABLE_PROPS agent?: DescriptHttpBlockDescriptionCallback; @@ -289,6 +289,7 @@ class HttpBlock< isError: block.isError, isRetryAllowed: block.isRetryAllowed, retryTimeout: block.retryTimeout, + getRetryStrategy: block.getRetryStrategy, body: null, ...( EVALUABLE_PROPS.reduce((ret, prop) => { diff --git a/lib/index.ts b/lib/index.ts index c17ec09..b011f78 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -6,7 +6,7 @@ import Logger, { EVENT } from './logger'; import type { LoggerEvent, LoggerInterface } from './logger'; import Cache, { CacheInterface } from './cache'; -import request, { type RequestOptions } from './request'; +import request, { type RequestOptions, type GetRetryStrategyParams } from './request'; import type { GenerateId, DescriptBlockDeps, DescriptBlockId } from './depsDomain'; import Block from './block'; import ArrayBlock from './arrayBlock'; @@ -15,6 +15,7 @@ import type { FunctionBlockDefinition } from './functionBlock'; import FunctionBlock from './functionBlock'; import HttpBlock from './httpBlock'; import FirstBlock from './firstBlock'; +import type { RetryStrategyInterface } from './retryStrategy'; import type { DescriptHttpBlockResult, @@ -202,6 +203,8 @@ export { DescriptHttpBlockDescription, DescriptHttpBlockQuery, DescriptHttpBlockQueryValue, + RetryStrategyInterface, + GetRetryStrategyParams, GenerateId, DescriptBlockId, InferResultFromBlock, diff --git a/lib/request.ts b/lib/request.ts index 261f115..a4362d8 100644 --- a/lib/request.ts +++ b/lib/request.ts @@ -20,13 +20,14 @@ import { EVENT } from './logger'; import type { Deffered } from './getDeferred'; import getDeferred from './getDeferred'; import type Cancel from './cancel'; -import type { DescriptError, Reason } from './error'; +import { DescriptError, Reason } from './error'; import { createError, ERROR_ID } from './error'; import is_plain_object from './isPlainObject'; import extend from './extend'; import type http from 'node:http'; import type { DescriptHttpResult, DescriptJSON } from './types'; +import { BaseRetryStrategy, RetryStrategyInterface, RetryStrategyRequest } from './retryStrategy'; // --------------------------------------------------------------------------------------------------------------- // @@ -80,6 +81,17 @@ export interface DescriptRequestOptions { bodyCompress?: ZlibOptions; agent?: HttpsAgent | HttpsAgentOptions | false | null; + + getRetryStrategy?: ({ + requestOptions, + logger, + }: GetRetryStrategyParams) => RetryStrategyInterface; +} + +export interface GetRetryStrategyParams { + requestOptions: RequestOptions; + logger: LoggerInterface; + request: RetryStrategyRequest; } export interface BlockRequestOptions { @@ -577,42 +589,28 @@ class DescriptRequest { async function request(options: DescriptRequestOptions, logger: LoggerInterface, cancel: Cancel): Promise { const requestOptions = new RequestOptions(options); - while (true) { + const request = async() => { const req = new DescriptRequest(requestOptions, logger, cancel); try { - const result = await req.start(); - - return result; - + return await req.start(); } catch (error) { if (error.error.statusCode === 429 || error.error.statusCode >= 500) { // Удаляем сокет, чтобы не залипать на отвечающем ошибкой бекэнде. req.destroyRequestSocket(); } - if (requestOptions.retries < requestOptions.maxRetries && requestOptions.isRetryAllowed?.(error, requestOptions)) { - requestOptions.retries++; + throw error; + } + }; - if (requestOptions.retryTimeout > 0) { - await waitFor(requestOptions.retryTimeout); - } + const retryStrategy = options.getRetryStrategy?.({ requestOptions, logger, request }) || new BaseRetryStrategy({ requestOptions, request }); - } else { - throw error; - } - } - } + return await retryStrategy.makeRequest(); } request.DEFAULT_OPTIONS = DEFAULT_OPTIONS; -function waitFor(timeout: number) { - return new Promise((resolve) => { - setTimeout(resolve, timeout); - }); -} - class ZstdDecompress extends Transform { receivedLength: number; receivedChunks: Array; diff --git a/lib/retryStrategy.ts b/lib/retryStrategy.ts new file mode 100644 index 0000000..aa26173 --- /dev/null +++ b/lib/retryStrategy.ts @@ -0,0 +1,59 @@ +import { DescriptError } from './error'; +import { RequestOptions } from './request'; +import type { DescriptHttpResult } from './types'; + +export type RetryStrategyRequest = () => Promise; + +export interface RetryStrategyInterface { + makeRequest: () => Promise; +} + +function waitFor(timeout: number) { + return new Promise((resolve) => { + setTimeout(resolve, timeout); + }); +} + +interface BaseRetryStrategyConstructorParams { + requestOptions: RequestOptions; + request: RetryStrategyRequest; +} + +export class BaseRetryStrategy implements RetryStrategyInterface { + requestOptions: RequestOptions; + request: RetryStrategyRequest; + + constructor({ requestOptions, request }: BaseRetryStrategyConstructorParams) { + this.request = request; + this.requestOptions = requestOptions; + } + + private async retry() { + this.requestOptions.retries++; + + if (this.requestOptions.retryTimeout > 0) { + await waitFor(this.requestOptions.retryTimeout); + } + + return this.makeRequest(); + } + + private isRetryAllowed(error: DescriptError) { + return ( + this.requestOptions.retries < this.requestOptions.maxRetries && + this.requestOptions.isRetryAllowed?.(error, this.requestOptions) + ); + } + + async makeRequest(): Promise { + try { + return await this.request(); + } catch (error) { + if (this.isRetryAllowed(error)) { + return this.retry(); + } else { + throw error; + } + } + } +} diff --git a/package-lock.json b/package-lock.json index 8240cda..b4385d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "descript", - "version": "4.0.18", + "version": "4.0.19", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "descript", - "version": "4.0.18", + "version": "4.0.19", "license": "MIT", "dependencies": { "@fengkx/zstd-napi": "^0.1.0" diff --git a/package.json b/package.json index 8d371fd..67d0093 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ }, "name": "descript", "description": "descript", - "version": "4.0.18", + "version": "4.0.19", "homepage": "https://github.com/descript-org/descript", "repository": { "type": "git", diff --git a/tests/request.test.ts b/tests/request.test.ts index 58b6a9f..885bde5 100644 --- a/tests/request.test.ts +++ b/tests/request.test.ts @@ -16,7 +16,8 @@ import https_ from 'https'; import http from 'http'; import fs_ from 'fs'; import path_ from 'path'; -import type { Cancel, LoggerInterface } from '../lib'; +import type { Cancel, LoggerInterface, RetryStrategyInterface, GetRetryStrategyParams } from '../lib'; +import { DescriptHttpResult } from 'lib/types'; // --------------------------------------------------------------------------------------------------------------- // @@ -1093,6 +1094,125 @@ describe('request', () => { }); + describe('getRetryStrategy', () => { + const PORT = 9000; + const CUSTOM_RETRY_ERROR_ID = 'CUSTOM_RETRY_ERROR'; + + const doRequest = getDoRequest({ + protocol: 'http:', + hostname: '127.0.0.1', + port: PORT, + pathname: '/', + }); + + const fake = new Server({ + module: http, + listen_options: { + port: PORT, + }, + }); + + beforeAll(() => Promise.all([ + fake.start(), + ])); + + afterAll(() => Promise.all([ + fake.stop(), + ])); + + class MyRetryStrategy implements RetryStrategyInterface { + retries: number; + maxRetries: number; + request: GetRetryStrategyParams['request']; + + constructor(request: GetRetryStrategyParams['request'], maxRetries: number) { + this.retries = 0; + this.request = request; + this.maxRetries = maxRetries; + } + + public async makeRequest(): Promise { + try { + return await this.request(); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (e) { + return this.retry(); + } + } + + private retry() { + this.retries++; + + if (this.retries < this.maxRetries) { + return this.makeRequest(); + } else { + throw de.error({ + id: CUSTOM_RETRY_ERROR_ID, + }); + } + } + } + + it('The custom retry strategy is called as expected', async() => { + const path = getPath(); + const statusCode = 404; + const content = 'Hello!'; + + fake.add(path, [ + { + statusCode: statusCode, + }, + { + statusCode: statusCode, + }, + { + statusCode: 200, + content: content, + }, + ]); + + let requestMock: any; + + const result = await doRequest({ + pathname: path, + getRetryStrategy: ({ request }) => { + if (!requestMock) { + requestMock = vi.fn(request); + request = requestMock; + }; + + return new MyRetryStrategy(request, 5); + }, + }); + + expect(requestMock).toHaveBeenCalledTimes(3); + expect(result.statusCode).toBe(200); + expect(result.body?.toString()).toBe(content); + }); + + it('The custom retry strategy throws its error', async() => { + const path = getPath(); + + fake.add(path, [ + { + statusCode: 404, + }, + ]); + + try { + await doRequest({ + pathname: path, + getRetryStrategy: ({ request }) => { + const instance = new MyRetryStrategy(request, 1); + return instance; + }, + }); + } catch ({ error }) { + expect(error.id).toBe(CUSTOM_RETRY_ERROR_ID); + } + }); + }); + describe('aborted request', () => { describe('no bytes sent', () => {