diff --git a/examples/example_cookies.ts b/examples/example_cookies.ts new file mode 100644 index 0000000..8f92d4b --- /dev/null +++ b/examples/example_cookies.ts @@ -0,0 +1,58 @@ +import got from 'got'; +import { CookieJar } from 'tough-cookie'; +const tunnel = require('tunnel'); +const parse = require('parse-headers') +const cookie = require('cookie') + +import { catchCloudflare } from '../src/index'; + +// example helper function +async function main() { + // cookie jar is required! + const cookieJar = new CookieJar(); + const options = { + // use "url: to describe path + url: 'http://soap2day.is', + cookieJar, + // either disable retry or remove status code 503 from retries + retry: 0, + + agent: { + https: tunnel.httpOverHttp({ + proxy: { + host: 'ip', + port: 1256, + + // Basic authorization for proxy server if necessary + proxyAuth: '', + + // Header fields for proxy server if necessary + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36', + }, + }, + }), + }, + }; + + let res: any; + try { + // success without cloudflare? + // const res = await cloudscraper({ + // method: 'GET', + // url: 'https://rlsbb.ru/support-us', + // }); + res = await got.get(options); + return res; + } catch (error) { + // success with cloudflare? + res = await catchCloudflare(error, options); + } + + return cookie.parse(parse(res.req._header).cookie); +} + +main() + .then((res) => console.log(res)) + .catch((err) => console.error('fail', err)); diff --git a/package-lock.json b/package-lock.json index 6cd75f8..4bb53bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2836,10 +2836,9 @@ } }, "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", - "dev": true + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" }, "cookie-signature": { "version": "1.0.6", @@ -3205,6 +3204,12 @@ "is-obj": "^1.0.0" } }, + "duplexer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", + "dev": true + }, "duplexer2": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", @@ -3728,6 +3733,32 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=", + "dev": true, + "requires": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + }, + "dependencies": { + "split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", + "dev": true, + "requires": { + "through": "2" + } + } + } + }, "exec-sh": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz", @@ -3882,6 +3913,12 @@ "vary": "~1.1.2" }, "dependencies": { + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "dev": true + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -4208,6 +4245,12 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", "dev": true }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=", + "dev": true + }, "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -6673,6 +6716,12 @@ "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=", "dev": true }, + "map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=", + "dev": true + }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -7067,6 +7116,12 @@ "propagate": "^2.0.0" } }, + "node-cleanup": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-cleanup/-/node-cleanup-2.1.2.tgz", + "integrity": "sha1-esGavSl+Caf3KnFUXZUbUX5N3iw=", + "dev": true + }, "node-emoji": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz", @@ -10941,6 +10996,11 @@ "callsites": "^3.0.0" } }, + "parse-headers": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.3.tgz", + "integrity": "sha512-QhhZ+DCCit2Coi2vmAKbq5RGTRcQUOE2+REgv8vdyu7MnYx2eZztegqtTx99TZ86GTIwqiy3+4nQTWZ2tgmdCA==" + }, "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", @@ -11010,6 +11070,15 @@ "pify": "^2.0.0" } }, + "pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "dev": true, + "requires": { + "through": "~2.3" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -11154,6 +11223,21 @@ "ipaddr.js": "1.9.1" } }, + "ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", + "dev": true, + "requires": { + "event-stream": "=3.3.4" + } + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -12582,6 +12666,15 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=", + "dev": true, + "requires": { + "duplexer": "~0.1.1" + } + }, "stream-combiner2": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz", @@ -12592,6 +12685,12 @@ "readable-stream": "^2.0.2" } }, + "string-argv": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.1.2.tgz", + "integrity": "sha512-mBqPGEOMNJKXRo7z0keX0wlAhbBAjilUdPW13nN0PecVryZxdHIeM7TqbsSUA7VYuS00HGC6mojP7DlQzfa9ZA==", + "dev": true + }, "string-length": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-length/-/string-length-3.1.0.tgz", @@ -13003,6 +13102,63 @@ } } }, + "tsc-watch": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/tsc-watch/-/tsc-watch-4.2.3.tgz", + "integrity": "sha512-M1Lo37+ggVfavGX3ObUVMz9QBH7moqq2RlmBdxnz6a6etwecetznZ/ZgYOi2c9HQ4Ki2qStj7V9J/gSf0rThig==", + "dev": true, + "requires": { + "cross-spawn": "^5.1.0", + "node-cleanup": "^2.1.2", + "ps-tree": "^1.2.0", + "string-argv": "^0.1.1", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + } + } + }, "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", @@ -13018,6 +13174,11 @@ "tslib": "^1.8.1" } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index 2f11bbe..add2661 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,11 @@ "semantic-release": "semantic-release" }, "dependencies": { + "cookie": "^0.4.1", "delay": "^4.3.0", - "got": "^11.0.3" + "got": "^11.0.3", + "parse-headers": "^2.0.3", + "tunnel": "0.0.6" }, "devDependencies": { "@pollyjs/adapter-fetch": "4.2.1", @@ -62,6 +65,7 @@ "semantic-release": "17.0.7", "tough-cookie": "4.0.0", "ts-jest": "25.4.0", + "tsc-watch": "^4.2.3", "typescript": "3.8.3" }, "jest": { diff --git a/src/catchCloudflare.ts b/src/catchCloudflare.ts index 2bd561a..06a14c3 100644 --- a/src/catchCloudflare.ts +++ b/src/catchCloudflare.ts @@ -1,8 +1,8 @@ -import crypto from 'crypto'; +import * as crypto from 'crypto'; import delay from 'delay'; import got, { Response, RequiredRetryOptions } from 'got'; -import https from 'https'; -import vm from 'vm'; +import * as https from 'https'; +import * as vm from 'vm'; import { URLSearchParams } from 'url'; const BUG_REPORT = `