From ac13a94abec40883667bf28741f407935723a346 Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:16:39 -0800 Subject: [PATCH 1/7] chore: bump CycleTLS version --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 8d4ace05..dd1a211a 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "docs:deploy": "yarn docs:generate && gh-pages -d docs", "format": "prettier --write src/**/*.ts examples/**/*.{ts,js,tsx}", "prepare": "husky install", - "test": "jest" + "test": "jest --runInBand" }, "dependencies": { "@sinclair/typebox": "^0.32.20", @@ -56,7 +56,7 @@ "tslib": "^2.5.2" }, "peerDependencies": { - "cycletls": "^2.0.4" + "cycletls": "^2.0.5" }, "peerDependenciesMeta": { "cycletls": { From 1902e33da9ec9c51634844be76dc9e60c1196e63 Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:17:41 -0800 Subject: [PATCH 2/7] feat: implement experimental xp-forwarded-for support --- src/auth-user.ts | 31 +++++++++------ src/auth.ts | 29 +++++++++++++- src/scraper.ts | 13 ++++++ src/xpff.ts | 100 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 src/xpff.ts diff --git a/src/auth-user.ts b/src/auth-user.ts index 17bed937..8cab38f6 100644 --- a/src/auth-user.ts +++ b/src/auth-user.ts @@ -9,6 +9,7 @@ import { Check } from '@sinclair/typebox/value'; import * as OTPAuth from 'otpauth'; import { FetchParameters } from './api-types'; import debug from 'debug'; +import { generateXPFFHeader } from './xpff'; const log = debug('twitter-scraper:auth-user'); @@ -310,26 +311,30 @@ export class TwitterUserAuth extends TwitterGuestAuth { } } - async installCsrfToken(headers: Headers): Promise { - const cookies = await this.getCookies(); - const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); - if (xCsrfToken) { - headers.set('x-csrf-token', xCsrfToken.value); - } - } - async installTo(headers: Headers): Promise { headers.set('authorization', `Bearer ${this.bearerToken}`); - const cookie = await this.getCookieString(); - headers.set('cookie', cookie); - if (this.guestToken) { - headers.set('x-guest-token', this.guestToken); - } headers.set( 'user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', ); + + if (this.guestToken) { + // Guest token is optional for authenticated users + headers.set('x-guest-token', this.guestToken); + } + await this.installCsrfToken(headers); + + if (this.options?.experimental?.xpff) { + const guestId = await this.guestId(); + if (guestId != null) { + const xpffHeader = await generateXPFFHeader(guestId); + headers.set('x-xp-forwarded-for', xpffHeader); + } + } + + const cookie = await this.getCookieString(); + headers.set('cookie', cookie); } private async initLogin(): Promise { diff --git a/src/auth.ts b/src/auth.ts index cbcf0cb9..163eb980 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -10,6 +10,7 @@ import { } from './rate-limit'; import { AuthenticationError } from './errors'; import debug from 'debug'; +import { generateXPFFHeader } from './xpff'; const log = debug('twitter-scraper:auth'); @@ -17,6 +18,9 @@ export interface TwitterAuthOptions { fetch: typeof fetch; transform: Partial; rateLimitStrategy: RateLimitStrategy; + experimental: { + xpff?: boolean; + }; } export interface TwitterAuth { @@ -33,6 +37,9 @@ export interface TwitterAuth { */ cookieJar(): CookieJar; + /** + * Returns the current cookies. + */ getCookies(): Promise; /** @@ -187,13 +194,25 @@ export class TwitterGuestAuth implements TwitterAuth { 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', ); + await this.installCsrfToken(headers); + + if (this.options?.experimental?.xpff) { + const guestId = await this.guestId(); + if (guestId != null) { + const xpffHeader = await generateXPFFHeader(guestId); + headers.set('x-xp-forwarded-for', xpffHeader); + } + } + + headers.set('cookie', await this.getCookieString()); + } + + async installCsrfToken(headers: Headers): Promise { const cookies = await this.getCookies(); const xCsrfToken = cookies.find((cookie) => cookie.key === 'ct0'); if (xCsrfToken) { headers.set('x-csrf-token', xCsrfToken.value); } - - headers.set('cookie', await this.getCookieString()); } protected async setCookie(key: string, value: string): Promise { @@ -238,6 +257,12 @@ export class TwitterGuestAuth implements TwitterAuth { : 'https://x.com'; } + protected async guestId(): Promise { + const cookies = await this.getCookies(); + const guestIdCookie = cookies.find((cookie) => cookie.key === 'guest_id'); + return guestIdCookie ? guestIdCookie.value : null; + } + /** * Updates the authentication state with a new guest token from the Twitter API. */ diff --git a/src/scraper.ts b/src/scraper.ts index ac351e0b..a837ef57 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -67,6 +67,16 @@ export interface ScraperOptions { * A handling strategy for rate limits (HTTP 429). */ rateLimitStrategy: RateLimitStrategy; + + /** + * Experimental features that may be added, changed, or removed at any time. Use with caution. + */ + experimental: { + /** + * Enables the generation of the `x-xp-forwarded-for` header on requests. This may resolve some errors. + */ + xpff: boolean; + }; } /** @@ -595,6 +605,9 @@ export class Scraper { fetch: this.options?.fetch, transform: this.options?.transform, rateLimitStrategy: this.options?.rateLimitStrategy, + experimental: { + xpff: this.options?.experimental?.xpff, + }, }; } diff --git a/src/xpff.ts b/src/xpff.ts new file mode 100644 index 00000000..9b0d13ea --- /dev/null +++ b/src/xpff.ts @@ -0,0 +1,100 @@ +import debug from 'debug'; + +const log = debug('twitter-scraper:xpff'); + +let isoCrypto: Crypto | null = null; + +function getCrypto(): Crypto { + if (isoCrypto != null) { + return isoCrypto; + } + + // In Node.js, the global `crypto` object is only available from v19.0.0 onwards. + // For earlier versions, we need to import the 'crypto' module. + if (typeof crypto === 'undefined') { + log('Global crypto is undefined, importing from crypto module...'); + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { webcrypto } = require('crypto'); + isoCrypto = webcrypto; + return webcrypto; + } + isoCrypto = crypto; + return crypto; +} + +async function sha256(message: string): Promise { + const msgBuffer = new TextEncoder().encode(message); + const hashBuffer = await getCrypto().subtle.digest('SHA-256', msgBuffer); + return new Uint8Array(hashBuffer); +} + +// https://stackoverflow.com/a/40031979 +function buf2hex(buffer: ArrayBuffer): string { + return [...new Uint8Array(buffer)] + .map((x) => x.toString(16).padStart(2, '0')) + .join(''); +} + +// Adapted from https://github.com/dsekz/twitter-x-xp-forwarded-for-header +export class XPFFHeaderGenerator { + constructor(private readonly seed: string) {} + + private async deriveKey(guestId: string): Promise { + const combined = `${this.seed}${guestId}`; + const result = await sha256(combined); + return result; + } + + async generateHeader(plaintext: string, guestId: string): Promise { + log(`Generating XPFF key for guest ID: ${guestId}`); + const key = await this.deriveKey(guestId); + const nonce = getCrypto().getRandomValues(new Uint8Array(12)); + const cipher = await getCrypto().subtle.importKey( + 'raw', + key, + { name: 'AES-GCM' }, + false, + ['encrypt'], + ); + const encrypted = await getCrypto().subtle.encrypt( + { + name: 'AES-GCM', + iv: nonce, + }, + cipher, + new TextEncoder().encode(plaintext), + ); + + // Combine nonce and encrypted data + const combined = new Uint8Array(nonce.length + encrypted.byteLength); + combined.set(nonce); + combined.set(new Uint8Array(encrypted), nonce.length); + const result = buf2hex(combined); + + log(`XPFF header generated for guest ID ${guestId}: ${result}`); + + return result; + } +} + +const xpffBaseKey = + '0e6be1f1e21ffc33590b888fd4dc81b19713e570e805d4e5df80a493c9571a05'; + +function xpffPlain(): string { + const timestamp = Date.now(); + return JSON.stringify({ + navigator_properties: { + hasBeenActive: 'true', + userAgent: + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36', + webdriver: 'false', + }, + created_at: timestamp, + }); +} + +export async function generateXPFFHeader(guestId: string): Promise { + const generator = new XPFFHeaderGenerator(xpffBaseKey); + const plaintext = xpffPlain(); + return generator.generateHeader(plaintext, guestId); +} From 915def7d7b4869509be49df112f74f29f5546c96 Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:20:33 -0800 Subject: [PATCH 3/7] chore: pass --forceExit to jest --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index dd1a211a..f9269439 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "docs:deploy": "yarn docs:generate && gh-pages -d docs", "format": "prettier --write src/**/*.ts examples/**/*.{ts,js,tsx}", "prepare": "husky install", - "test": "jest --runInBand" + "test": "jest --runInBand --forceExit" }, "dependencies": { "@sinclair/typebox": "^0.32.20", From bb86178acd018dd5230fce5c24e948843c1efb4c Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 16:59:48 -0800 Subject: [PATCH 4/7] feat: add x-client-transaction-id generation --- package.json | 6 +- src/api.ts | 15 +++- src/auth-user.ts | 10 +++ src/auth.ts | 3 +- src/scraper.ts | 5 ++ src/test-utils.ts | 4 ++ src/xctxid.ts | 175 ++++++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 108 +++++++++++++++++++++++++++- 8 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 src/xctxid.ts diff --git a/package.json b/package.json index f9269439..61a877db 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "scraper", "crawler" ], - "version": "0.19.1", + "version": "0.20.0", "main": "dist/default/cjs/index.js", "types": "./dist/types/index.d.ts", "exports": { @@ -50,10 +50,12 @@ "debug": "^4.4.1", "headers-polyfill": "^3.1.2", "json-stable-stringify": "^1.0.2", + "linkedom": "^0.18.12", "otpauth": "^9.2.2", "set-cookie-parser": "^2.6.0", "tough-cookie": "^4.1.2", - "tslib": "^2.5.2" + "tslib": "^2.5.2", + "x-client-transaction-id": "^0.1.9" }, "peerDependencies": { "cycletls": "^2.0.5" diff --git a/src/api.ts b/src/api.ts index 3261afa0..c545c808 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,10 +1,11 @@ import { FetchParameters } from './api-types'; -import { TwitterAuth } from './auth'; +import { TwitterAuth, TwitterGuestAuth } from './auth'; import { ApiError } from './errors'; import { Platform, PlatformExtensions } from './platform'; import { updateCookieJar } from './requests'; import { Headers } from 'headers-polyfill'; import debug from 'debug'; +import { generateTransactionId } from './xctxid'; const log = debug('twitter-scraper:api'); @@ -63,6 +64,18 @@ export async function requestApi( await auth.installTo(headers, url); await platform.randomizeCiphers(); + if ( + auth instanceof TwitterGuestAuth && + auth.options?.experimental?.xClientTransactionId + ) { + const transactionId = await generateTransactionId( + url, + auth.fetch.bind(auth), + method, + ); + headers.set('x-client-transaction-id', transactionId); + } + let res: Response; do { const fetchParameters: FetchParameters = [ diff --git a/src/auth-user.ts b/src/auth-user.ts index 8cab38f6..a54de23a 100644 --- a/src/auth-user.ts +++ b/src/auth-user.ts @@ -10,6 +10,7 @@ import * as OTPAuth from 'otpauth'; import { FetchParameters } from './api-types'; import debug from 'debug'; import { generateXPFFHeader } from './xpff'; +import { generateTransactionId } from './xctxid'; const log = debug('twitter-scraper:auth-user'); @@ -628,6 +629,15 @@ export class TwitterUserAuth extends TwitterGuestAuth { }); await this.installTo(headers); + if (this.options?.experimental?.xClientTransactionId) { + const transactionId = await generateTransactionId( + onboardingTaskUrl, + this.fetch.bind(this), + 'POST', + ); + headers.set('x-client-transaction-id', transactionId); + } + let res: Response; do { const fetchParameters: FetchParameters = [ diff --git a/src/auth.ts b/src/auth.ts index 163eb980..69f1cd57 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -19,6 +19,7 @@ export interface TwitterAuthOptions { transform: Partial; rateLimitStrategy: RateLimitStrategy; experimental: { + xClientTransactionId?: boolean; xpff?: boolean; }; } @@ -126,7 +127,7 @@ export class TwitterGuestAuth implements TwitterAuth { constructor( bearerToken: string, - protected readonly options?: Partial, + readonly options?: Partial, ) { this.fetch = withTransform(options?.fetch ?? fetch, options?.transform); this.rateLimitStrategy = diff --git a/src/scraper.ts b/src/scraper.ts index a837ef57..4b259d0e 100644 --- a/src/scraper.ts +++ b/src/scraper.ts @@ -72,6 +72,10 @@ export interface ScraperOptions { * Experimental features that may be added, changed, or removed at any time. Use with caution. */ experimental: { + /** + * Enables the generation of the `x-client-transaction-id` header on requests. This may resolve some errors. + */ + xClientTransactionId: boolean; /** * Enables the generation of the `x-xp-forwarded-for` header on requests. This may resolve some errors. */ @@ -606,6 +610,7 @@ export class Scraper { transform: this.options?.transform, rateLimitStrategy: this.options?.rateLimitStrategy, experimental: { + xClientTransactionId: this.options?.experimental?.xClientTransactionId, xpff: this.options?.experimental?.xpff, }, }; diff --git a/src/test-utils.ts b/src/test-utils.ts index 5d18e7f3..ba76f037 100644 --- a/src/test-utils.ts +++ b/src/test-utils.ts @@ -55,6 +55,10 @@ export async function getScraper( return [input, init]; }, }, + experimental: { + xClientTransactionId: true, + xpff: true, + }, }); if (options.authMethod === 'password') { diff --git a/src/xctxid.ts b/src/xctxid.ts new file mode 100644 index 00000000..65c44e33 --- /dev/null +++ b/src/xctxid.ts @@ -0,0 +1,175 @@ +import fetch from 'cross-fetch'; +import debug from 'debug'; + +const log = debug('twitter-scraper:xctxid'); + +// @ts-expect-error import type annotation ("the current file is a CommonJS module") +type LinkeDOM = typeof import('linkedom'); + +let linkedom: LinkeDOM | null = null; +function linkedomImport(): LinkeDOM { + if (!linkedom) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const mod = require('linkedom'); + linkedom = mod; + return mod; + } + return linkedom; +} + +async function parseHTML(html: string): Promise { + if (typeof window !== 'undefined') { + const { defaultView } = new DOMParser().parseFromString(html, 'text/html'); + if (!defaultView) { + throw new Error('Failed to get defaultView from parsed HTML.'); + } + return defaultView; + } else { + const { DOMParser } = linkedomImport(); + return new DOMParser().parseFromString(html, 'text/html').defaultView; + } +} + +// Copied from https://github.com/Lqm1/x-client-transaction-id/blob/main/utils.ts with minor tweaks +async function handleXMigration(fetchFn: typeof fetch): Promise { + // Set headers to mimic a browser request + const headers = { + accept: + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'accept-language': 'ja', + 'cache-control': 'no-cache', + pragma: 'no-cache', + priority: 'u=0, i', + 'sec-ch-ua': + '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'user-agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', + }; + + // Fetch X.com homepage + const response = await fetchFn('https://x.com', { + headers, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch X homepage: ${response.statusText}`); + } + + const htmlText = await response.text(); + + // Parse HTML using linkedom + let dom = await parseHTML(htmlText); + let document = dom.window.document; + + // Check for migration redirection links + const migrationRedirectionRegex = new RegExp( + '(http(?:s)?://(?:www\\.)?(twitter|x){1}\\.com(/x)?/migrate([/?])?tok=[a-zA-Z0-9%\\-_]+)+', + 'i', + ); + + const metaRefresh = document.querySelector("meta[http-equiv='refresh']"); + const metaContent = metaRefresh + ? metaRefresh.getAttribute('content') || '' + : ''; + + const migrationRedirectionUrl = + migrationRedirectionRegex.exec(metaContent) || + migrationRedirectionRegex.exec(htmlText); + + if (migrationRedirectionUrl) { + // Follow redirection URL + const redirectResponse = await fetch(migrationRedirectionUrl[0]); + + if (!redirectResponse.ok) { + throw new Error( + `Failed to follow migration redirection: ${redirectResponse.statusText}`, + ); + } + + const redirectHtml = await redirectResponse.text(); + dom = await parseHTML(redirectHtml); + document = dom.window.document; + } + + // Handle migration form if present + const migrationForm = + document.querySelector("form[name='f']") || + document.querySelector("form[action='https://x.com/x/migrate']"); + + if (migrationForm) { + const url = + migrationForm.getAttribute('action') || 'https://x.com/x/migrate'; + const method = migrationForm.getAttribute('method') || 'POST'; + + // Collect form input fields + const requestPayload = new FormData(); + + const inputFields = migrationForm.querySelectorAll('input'); + for (const element of Array.from(inputFields)) { + const name = element.getAttribute('name'); + const value = element.getAttribute('value'); + if (name && value) { + requestPayload.append(name, value); + } + } + + // Submit form using POST request + const formResponse = await fetch(url, { + method: method, + body: requestPayload, + headers, + }); + + if (!formResponse.ok) { + throw new Error( + `Failed to submit migration form: ${formResponse.statusText}`, + ); + } + + const formHtml = await formResponse.text(); + dom = await parseHTML(formHtml); + document = dom.window.document; + } + + // Return final DOM document + return document; +} + +let ClientTransaction: + | typeof import('x-client-transaction-id')['ClientTransaction'] + | null = null; +function clientTransaction(): typeof import('x-client-transaction-id')['ClientTransaction'] { + if (!ClientTransaction) { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const mod = require('x-client-transaction-id'); + const ctx = + mod.ClientTransaction as typeof import('x-client-transaction-id')['ClientTransaction']; + ClientTransaction = ctx; + return ctx; + } + return ClientTransaction; +} + +export async function generateTransactionId( + url: string, + fetchFn: typeof fetch, + method: 'GET' | 'POST', +) { + const parsedUrl = new URL(url); + const path = parsedUrl.pathname; + + log(`Generating transaction ID for ${method} ${path}`); + const document = await handleXMigration(fetchFn); + const transaction = await clientTransaction().create(document); + const transactionId = await transaction.generateTransactionId(method, path); + log(`Transaction ID: ${transactionId}`); + + return transactionId; +} diff --git a/yarn.lock b/yarn.lock index 37dc8712..9ff2ae96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1873,6 +1873,11 @@ bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +boolbase@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" + integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -2283,6 +2288,27 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +css-select@^5.1.0: + version "5.2.2" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.2.2.tgz#01b6e8d163637bb2dd6c982ca4ed65863682786e" + integrity sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + +css-what@^6.1.0: + version "6.2.2" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.2.2.tgz#cdcc8f9b6977719fdfbd1de7aec24abf756b9dea" + integrity sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA== + +cssom@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" + integrity sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw== + cycletls@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/cycletls/-/cycletls-2.0.4.tgz#8ae978f05a43ba9aba2d3c19d89630cc736868ab" @@ -2425,6 +2451,36 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== + +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + +domutils@^3.0.1, domutils@^3.2.1: + version "3.2.2" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.2.2.tgz#edbfe2b668b0c1d97c24baf0f1062b132221bc78" + integrity sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + dot-prop@^5.1.0: version "5.3.0" resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88" @@ -2481,11 +2537,16 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== -entities@^4.4.0: +entities@^4.2.0, entities@^4.4.0: version "4.5.0" resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== +entities@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-6.0.1.tgz#c28c34a43379ca7f61d074130b2f5f7020a30694" + integrity sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g== + env-paths@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" @@ -3289,6 +3350,21 @@ html-escaper@^2.0.0: resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== +html-escaper@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-3.0.3.tgz#4d336674652beb1dcbc29ef6b6ba7f6be6fdfed6" + integrity sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ== + +htmlparser2@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-10.0.0.tgz#77ad249037b66bf8cc99c6e286ef73b83aeb621d" + integrity sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.2.1" + entities "^6.0.0" + https-proxy-agent@^7.0.2: version "7.0.4" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" @@ -4078,6 +4154,17 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +linkedom@^0.18.12, linkedom@^0.18.9: + version "0.18.12" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.18.12.tgz#a8b1a1942b567dcb1888093df311055da1349a14" + integrity sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^10.0.0" + uhyphen "^0.2.0" + linkify-it@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421" @@ -4511,6 +4598,13 @@ npm-run-path@^5.1.0: dependencies: path-key "^4.0.0" +nth-check@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" + integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== + dependencies: + boolbase "^1.0.0" + object-assign@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -5465,6 +5559,11 @@ uc.micro@^2.0.0, uc.micro@^2.1.0: resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee" integrity sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A== +uhyphen@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.2.0.tgz#8fdf0623314486e020a3c00ee5cc7a12fe722b81" + integrity sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA== + undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" @@ -5630,6 +5729,13 @@ ws@^8.17.0: resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.3.tgz#b56b88abffde62791c639170400c93dcb0c95472" integrity sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg== +x-client-transaction-id@^0.1.9: + version "0.1.9" + resolved "https://registry.yarnpkg.com/x-client-transaction-id/-/x-client-transaction-id-0.1.9.tgz#3bb0359268a481f66edf39c0aaa195d73195e849" + integrity sha512-CES4zgkJ0wbfFWm0qgdKphthyb+L7lVHymgOY15v6ivcWSx5p9lp5kzAed+BuqJSP7bS0GbQyJ16ONkRthgsUw== + dependencies: + linkedom "^0.18.9" + y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" From ecfa6a2366dff9b7a116cc8c290ee4d46535e7d2 Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 17:01:57 -0800 Subject: [PATCH 5/7] chore: update comment --- src/xctxid.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xctxid.ts b/src/xctxid.ts index 65c44e33..90e080b8 100644 --- a/src/xctxid.ts +++ b/src/xctxid.ts @@ -30,7 +30,7 @@ async function parseHTML(html: string): Promise { } } -// Copied from https://github.com/Lqm1/x-client-transaction-id/blob/main/utils.ts with minor tweaks +// Copied from https://github.com/Lqm1/x-client-transaction-id/blob/main/utils.ts with minor tweaks to support us passing a custom fetch function async function handleXMigration(fetchFn: typeof fetch): Promise { // Set headers to mimic a browser request const headers = { From 0fc4bb5734ebdc35bacddb79ffe8bd8eb4f7f99f Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 17:13:42 -0800 Subject: [PATCH 6/7] chore: use experimental options in examples for validation --- examples/node-integration/package.json | 3 +- examples/node-integration/src/index.cjs | 10 +- examples/node-integration/src/index.mjs | 10 +- examples/node-integration/yarn.lock | 133 +++++++++++++++++++++++- 4 files changed, 152 insertions(+), 4 deletions(-) diff --git a/examples/node-integration/package.json b/examples/node-integration/package.json index 35c5d72e..f62dcf1d 100644 --- a/examples/node-integration/package.json +++ b/examples/node-integration/package.json @@ -6,7 +6,8 @@ "start:cjs": "node src/index.cjs" }, "dependencies": { - "@the-convocation/twitter-scraper": "file:../../" + "@the-convocation/twitter-scraper": "file:../../", + "dotenv": "^17.2.3" }, "devDependencies": {} } diff --git a/examples/node-integration/src/index.cjs b/examples/node-integration/src/index.cjs index b1647253..68f29006 100644 --- a/examples/node-integration/src/index.cjs +++ b/examples/node-integration/src/index.cjs @@ -1,7 +1,10 @@ /* eslint-disable @typescript-eslint/no-var-requires */ const assert = require('node:assert'); +const dotenv = require('dotenv'); const { Scraper } = require('@the-convocation/twitter-scraper'); +dotenv.config({ path: '../../.env.local' }); + // Debug logging to show that the Node.js build is being loaded console.log( `Loaded @the-convocation/twitter-scraper from ${require.resolve( @@ -21,7 +24,12 @@ const email = process.env['TWITTER_EMAIL']; assert(username && password && email); -const scraper = new Scraper(); +const scraper = new Scraper({ + experimental: { + xClientTransactionId: true, + xpff: true, + }, +}); const main = async () => { await scraper.login(username, password, email); diff --git a/examples/node-integration/src/index.mjs b/examples/node-integration/src/index.mjs index d4d5958b..96c0204a 100644 --- a/examples/node-integration/src/index.mjs +++ b/examples/node-integration/src/index.mjs @@ -1,6 +1,9 @@ import assert from 'node:assert'; +import dotenv from 'dotenv'; import { Scraper } from '@the-convocation/twitter-scraper'; +dotenv.config({ path: '../../.env.local' }); + // Debug logging to show that the Node.js build is being loaded console.log( `Loaded @the-convocation/twitter-scraper from ${import.meta.resolve( @@ -20,7 +23,12 @@ const email = process.env['TWITTER_EMAIL']; assert(username && password && email); -const scraper = new Scraper(); +const scraper = new Scraper({ + experimental: { + xClientTransactionId: true, + xpff: true, + }, +}); await scraper.login(username, password, email); const tweet = await scraper.getTweet('1585338303800578049'); diff --git a/examples/node-integration/yarn.lock b/examples/node-integration/yarn.lock index 3bb4f696..e040da68 100644 --- a/examples/node-integration/yarn.lock +++ b/examples/node-integration/yarn.lock @@ -13,16 +13,24 @@ integrity sha512-m+A3zFSI87TCtoz6vQCSnd+t/kDKL78JmzhKYkON+7SnHSa+794qraIVpm6ozFGK+5svnVOt1LJ7BUEhGkIvgg== "@the-convocation/twitter-scraper@file:../..": - version "0.12.0" + version "0.20.0" dependencies: "@sinclair/typebox" "^0.32.20" cross-fetch "^4.0.0-alpha.5" + debug "^4.4.1" headers-polyfill "^3.1.2" json-stable-stringify "^1.0.2" + linkedom "^0.18.12" otpauth "^9.2.2" set-cookie-parser "^2.6.0" tough-cookie "^4.1.2" tslib "^2.5.2" + x-client-transaction-id "^0.1.9" + +boolbase@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" + integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== call-bind@^1.0.5: version "1.0.7" @@ -42,6 +50,34 @@ cross-fetch@^4.0.0-alpha.5: dependencies: node-fetch "^2.6.12" +css-select@^5.1.0: + version "5.2.2" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.2.2.tgz#01b6e8d163637bb2dd6c982ca4ed65863682786e" + integrity sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw== + dependencies: + boolbase "^1.0.0" + css-what "^6.1.0" + domhandler "^5.0.2" + domutils "^3.0.1" + nth-check "^2.0.1" + +css-what@^6.1.0: + version "6.2.2" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.2.2.tgz#cdcc8f9b6977719fdfbd1de7aec24abf756b9dea" + integrity sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA== + +cssom@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" + integrity sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw== + +debug@^4.4.1: + version "4.4.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" + integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== + dependencies: + ms "^2.1.3" + define-data-property@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" @@ -51,6 +87,51 @@ define-data-property@^1.1.4: es-errors "^1.3.0" gopd "^1.0.1" +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" + integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== + +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + +domutils@^3.0.1, domutils@^3.2.1: + version "3.2.2" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.2.2.tgz#edbfe2b668b0c1d97c24baf0f1062b132221bc78" + integrity sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + +dotenv@^17.2.3: + version "17.2.3" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-17.2.3.tgz#ad995d6997f639b11065f419a22fabf567cdb9a2" + integrity sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w== + +entities@^4.2.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + +entities@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/entities/-/entities-6.0.1.tgz#c28c34a43379ca7f61d074130b2f5f7020a30694" + integrity sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g== + es-define-property@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.0.tgz#c7faefbdff8b2696cf5f46921edfb77cc4ba3845" @@ -115,6 +196,21 @@ headers-polyfill@^3.1.2: resolved "https://registry.yarnpkg.com/headers-polyfill/-/headers-polyfill-3.3.0.tgz#67c6ef7b72d4c8cac832ad5936f5b3a56e7b705a" integrity sha512-5e57etwBpNcDc0b6KCVWEh/Ro063OxPvzVimUdM0/tsYM/T7Hfy3kknIGj78SFTOhNd8AZY41U8mOHoO4LzmIQ== +html-escaper@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-3.0.3.tgz#4d336674652beb1dcbc29ef6b6ba7f6be6fdfed6" + integrity sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ== + +htmlparser2@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-10.0.0.tgz#77ad249037b66bf8cc99c6e286ef73b83aeb621d" + integrity sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.2.1" + entities "^6.0.0" + isarray@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" @@ -135,6 +231,22 @@ jsonify@^0.0.1: resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.1.tgz#2aa3111dae3d34a0f151c63f3a45d995d9420978" integrity sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg== +linkedom@^0.18.12, linkedom@^0.18.9: + version "0.18.12" + resolved "https://registry.yarnpkg.com/linkedom/-/linkedom-0.18.12.tgz#a8b1a1942b567dcb1888093df311055da1349a14" + integrity sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q== + dependencies: + css-select "^5.1.0" + cssom "^0.5.0" + html-escaper "^3.0.3" + htmlparser2 "^10.0.0" + uhyphen "^0.2.0" + +ms@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + node-fetch@^2.6.12: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" @@ -142,6 +254,13 @@ node-fetch@^2.6.12: dependencies: whatwg-url "^5.0.0" +nth-check@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" + integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== + dependencies: + boolbase "^1.0.0" + object-keys@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" @@ -211,6 +330,11 @@ tslib@^2.5.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== +uhyphen@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/uhyphen/-/uhyphen-0.2.0.tgz#8fdf0623314486e020a3c00ee5cc7a12fe722b81" + integrity sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA== + universalify@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.2.0.tgz#6451760566fa857534745ab1dde952d1b1761be0" @@ -236,3 +360,10 @@ whatwg-url@^5.0.0: dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0" + +x-client-transaction-id@^0.1.9: + version "0.1.9" + resolved "https://registry.yarnpkg.com/x-client-transaction-id/-/x-client-transaction-id-0.1.9.tgz#3bb0359268a481f66edf39c0aaa195d73195e849" + integrity sha512-CES4zgkJ0wbfFWm0qgdKphthyb+L7lVHymgOY15v6ivcWSx5p9lp5kzAed+BuqJSP7bS0GbQyJ16ONkRthgsUw== + dependencies: + linkedom "^0.18.9" From c90b72c987c6f287cc1cd8eb42b630c0d3cb92ca Mon Sep 17 00:00:00 2001 From: karashiiro <49822414+karashiiro@users.noreply.github.com> Date: Sun, 9 Nov 2025 17:36:04 -0800 Subject: [PATCH 7/7] feat: warn and continue if guest token activation fails --- src/auth-user.ts | 9 --------- src/auth.ts | 16 ++++++++++------ 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/auth-user.ts b/src/auth-user.ts index a54de23a..0221bd9c 100644 --- a/src/auth-user.ts +++ b/src/auth-user.ts @@ -596,14 +596,6 @@ export class TwitterUserAuth extends TwitterGuestAuth { } log(`Making POST request to ${onboardingTaskUrl}`); - - const token = this.guestToken; - if (token == null) { - throw new AuthenticationError( - 'Authentication token is null or undefined.', - ); - } - const headers = new Headers({ accept: '*/*', 'accept-language': 'en-US,en;q=0.9', @@ -622,7 +614,6 @@ export class TwitterUserAuth extends TwitterGuestAuth { 'sec-fetch-site': 'same-origin', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', - 'x-guest-token': token, 'x-twitter-auth-type': 'OAuth2Client', 'x-twitter-active-user': 'yes', 'x-twitter-client-language': 'en', diff --git a/src/auth.ts b/src/auth.ts index 69f1cd57..16eed233 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -181,15 +181,11 @@ export class TwitterGuestAuth implements TwitterAuth { await this.updateGuestToken(); } - const token = this.guestToken; - if (token == null) { - throw new AuthenticationError( - 'Authentication token is null or undefined.', - ); + if (this.guestToken) { + headers.set('x-guest-token', this.guestToken); } headers.set('authorization', `Bearer ${this.bearerToken}`); - headers.set('x-guest-token', token); headers.set( 'user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36', @@ -268,6 +264,14 @@ export class TwitterGuestAuth implements TwitterAuth { * Updates the authentication state with a new guest token from the Twitter API. */ protected async updateGuestToken() { + try { + await this.updateGuestTokenCore(); + } catch (err) { + log('Failed to update guest token; this may cause issues:', err); + } + } + + private async updateGuestTokenCore() { const guestActivateUrl = 'https://api.x.com/1.1/guest/activate.json'; const headers = new Headers({