NewsWebScraper/proxyFetch.js at main · Algorithm-Allies/NewsWebScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
require("dotenv").config();
const { HttpsProxyAgent } = require("https-proxy-agent");

// Array of proxy objects
const proxies = [
  { ip: process.env.PROXY_IP_1, port: process.env.PORT_1 },
  { ip: process.env.PROXY_IP_2, port: process.env.PORT_2 },
  { ip: process.env.PROXY_IP_3, port: process.env.PORT_3 },
  { ip: process.env.PROXY_IP_4, port: process.env.PORT_4 },
  { ip: process.env.PROXY_IP_5, port: process.env.PORT_5 },
  { ip: process.env.PROXY_IP_6, port: process.env.PORT_6 },
  { ip: process.env.PROXY_IP_7, port: process.env.PORT_7 },
  { ip: process.env.PROXY_IP_8, port: process.env.PORT_8 },
  { ip: process.env.PROXY_IP_9, port: process.env.PORT_9 },
  { ip: process.env.PROXY_IP_10, port: process.env.PORT_10 },
];

const headers = [
  {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-us",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Linux; Android 10; SM-G981B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.210 Mobile Safari/537.36",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Android 10; Mobile; rv:88.0) Gecko/88.0 Firefox/88.0",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.818.62 Safari/537.36 Edg/90.0.818.46",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (iPhone; CPU iPhone OS 14_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Mobile/15E148 Safari/604.1",
    Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 OPR/60.0.3255.170",
    Accept:
      "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Language": "en-US,en;q=0.9",
  },
  {
    "User-Agent":
      "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko",
    Accept: "text/html, application/xhtml+xml, image/jxr, */*",
    "Accept-Language": "en-US,en;q=0.5",
  },
];

// Function to pick a random proxy object from list.
function getRandomIndex() {
  const randIndex = Math.floor(Math.random() * proxies.length);
  return randIndex;
}

// Function to make fetch request using random proxy.
async function fetchWithProxy(url) {
  try {
    const delay = Math.floor(Math.random() * 10) + 1;
    await new Promise((resolve) => setTimeout(resolve, delay * 1000));
    const index = getRandomIndex();

    const ip = proxies[index].ip;
    const port = proxies[index].port;
    const header = headers[index];

    const proxyAgent = new HttpsProxyAgent(`https://${ip}:${port}`);

    const response = await fetch(url, {
      headers: header,
      agent: proxyAgent,
      method: "GET",
      cache: "no-cache",
      signal: AbortSignal.timeout(15000),
    });
    return await response.text();
  } catch (e) {
    console.error("Proxy Request  Failed:", e);
  }
}

// Function to make fetch request using random proxy.
async function fetchWithProxyTracy(url) {
  try {
    const delay = Math.floor(Math.random() * 10) + 1;
    await new Promise((resolve) => setTimeout(resolve, delay * 1000));
    const index = getRandomIndex();

    const ip = proxies[index].ip;
    const port = proxies[index].port;
    const header = headers[index];

    const proxyAgent = new HttpsProxyAgent(`https://${ip}:${port}`);

    const response = await fetch(url, {
      headers: header,
      agent: proxyAgent,
      method: "GET",
      cache: "no-cache",
      signal: AbortSignal.timeout(15000),
    });
    if (typeof response == "string") {
      return response;
    }
    return response.text();
  } catch (e) {
    console.error("Proxy Request  Failed:", e);
  }
}

module.exports = { fetchWithProxy, fetchWithProxyTracy };