From f11eeb5254675838013b91bedb5be104e313b710 Mon Sep 17 00:00:00 2001 From: mei23 Date: Fri, 23 Jul 2021 14:01:00 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=9E=E3=83=AB=E3=83=81=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=B9=E3=82=BF=E3=83=B3=E3=82=B9=E5=AF=BE=E5=BF=9C=E3=81=AA?= =?UTF-8?q?=E3=81=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 8 ++++---- src/client.ts | 13 +++++++++++++ src/general.ts | 14 +++----------- src/plugins/amazon.ts | 12 +++--------- 4 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 src/client.ts diff --git a/package.json b/package.json index 5be5d083..f765bb40 100644 --- a/package.json +++ b/package.json @@ -14,18 +14,18 @@ }, "devDependencies": { "@types/debug": "0.0.30", - "@types/express": "4.16.0", + "@types/express": "4.17.3", "@types/html-entities": "1.2.16", "@types/mocha": "5.2.2", - "@types/node": "10.3.4", + "@types/node": "16.4.1", "@types/request": "2.47.1", "@types/request-promise-native": "1.0.15", "express": "4.16.3", "mocha": "5.2.0", - "typescript": "2.9.2" + "typescript": "4.3.5" }, "dependencies": { - "cheerio-httpcli": "0.7.4", + "cheerio-httpcli": "0.8.2", "debug": "3.1.0", "escape-regexp": "0.0.1", "html-entities": "1.2.1", diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 00000000..33b2ab49 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,13 @@ +import { version } from '../package.json'; +import * as client from 'cheerio-httpcli'; + +client.set('headers', { + 'User-Agent': `SummalyBot/${version}` +}); +client.set('referer', false); +client.set('timeout', 20000); +client.set('maxDataSize', 10 * 1024 * 1024); + +export function createInstance() { + return client.fork(); +} diff --git a/src/general.ts b/src/general.ts index b4e03dea..74600c00 100644 --- a/src/general.ts +++ b/src/general.ts @@ -1,4 +1,3 @@ -import { name, version } from '../package.json'; import * as URL from 'url'; import * as request from 'request'; import nullOrEmpty from './utils/null-or-empty'; @@ -8,21 +7,14 @@ import cleanupTitle from './utils/cleanup-title'; import { AllHtmlEntities } from 'html-entities'; const entities = new AllHtmlEntities(); -import * as client from 'cheerio-httpcli'; - -// 単一インスタンスなのでamazonと値を揃えないといけない -client.set('headers', { - 'User-Agent': `SummalyBot/${version}` -}); -client.set('referer', false); -client.set('timeout', 10000); -client.set('maxDataSize', 10 * 1024 * 1024); - +import { createInstance } from './client'; import Summary from './summary'; export default async (url: URL.Url, lang: string = null): Promise => { if (lang && !lang.match(/^[\w-]+(\s*,\s*[\w-]+)*$/)) lang = null; + const client = createInstance(); + client.set('headers', { 'Accept-Language': lang }); diff --git a/src/plugins/amazon.ts b/src/plugins/amazon.ts index a6067c35..d15e5ea3 100644 --- a/src/plugins/amazon.ts +++ b/src/plugins/amazon.ts @@ -1,15 +1,7 @@ -import { name, version } from '../../package.json'; import * as URL from 'url'; -import * as client from 'cheerio-httpcli'; +import { createInstance } from '../client'; import summary from '../summary'; -client.set('headers', { - 'User-Agent': `SummalyBot/${version}` -}); -client.set('referer', false); -client.set('timeout', 10000); -client.set('maxDataSize', 10 * 1024 * 1024); - export function test(url: URL.Url): boolean { return url.hostname === 'www.amazon.com' || url.hostname === 'www.amazon.co.jp' || @@ -28,6 +20,8 @@ export function test(url: URL.Url): boolean { } export async function summarize(url: URL.Url): Promise { + const client = createInstance(); + const res = await client.fetch(url.href); const $ = res.$;