diff --git a/lib/remote.js b/lib/remote.js index f1da5b3..66d1e91 100644 --- a/lib/remote.js +++ b/lib/remote.js @@ -4,7 +4,9 @@ const path = require('path'); const fs = require('fs-extra'); const unzip = require('adm-zip'); const config = require('./config'); -const axios = require('axios'); +const fetch = require('node-fetch'); +const { HttpsProxyAgent } = require('https-proxy-agent'); +const { getProxyForUrl } = require('proxy-from-env'); // Downloads the zip file from github and extracts it to folder exports.download = (loc, lang) => { @@ -12,21 +14,29 @@ exports.download = (loc, lang) => { const suffix = (lang === 'en' ? '' : '.' + lang); const url = config.get().repositoryBase + suffix + '.zip'; const folderName = path.join(loc, 'pages' + suffix); - const REQUEST_TIMEOUT = 10000; - - return axios({ - method: 'get', - url: url, - responseType: 'stream', - headers: { 'User-Agent' : 'tldr-node-client' }, - timeout: REQUEST_TIMEOUT, - }).then((response) => { + + const headers = { 'User-Agent' : 'tldr-node-client' }; + const fetchOptions = { headers }; + + const proxy = getProxyForUrl(url); + if (proxy) { + fetchOptions.agent = new HttpsProxyAgent(proxy, { headers }); + } + + return fetch(url, fetchOptions).then((response) => { return new Promise((resolve, reject) => { - let fileName = path.join(loc, 'download_' + lang + '.zip'); + if (!response.ok) { + reject(new Error( + `Fetch \`${url}\` failed, status = ${response.status}` + )); + return; + } + + const fileName = path.join(loc, 'download_' + lang + '.zip'); const writer = fs.createWriteStream(fileName); - response.data.pipe(writer); - + response.body.pipe(writer); + writer.on('finish', () => { writer.end(); const zip = new unzip(fileName); @@ -41,4 +51,4 @@ exports.download = (loc, lang) => { }).catch((err) => { return Promise.reject(err); }); -}; \ No newline at end of file +}; diff --git a/package-lock.json b/package-lock.json index a0bd5f3..bc309e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,16 +10,18 @@ "license": "MIT", "dependencies": { "adm-zip": "^0.5.10", - "axios": "^0.21.1", "chalk": "^4.1.0", "commander": "^6.1.0", "fs-extra": "^9.0.1", "glob": "^7.1.6", + "https-proxy-agent": "^7.0.2", "lodash": "^4.17.20", "marked": "^4.0.10", "ms": "^2.1.2", "natural": "^2.1.5", - "ora": "^5.1.0" + "node-fetch": "^2.7.0", + "ora": "^5.1.0", + "proxy-from-env": "^1.1.0" }, "bin": { "tldr": "bin/tldr" @@ -949,6 +951,17 @@ "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.3.tgz", "integrity": "sha512-LTwJcUGRzjQCR6uQCXMEUkoecezmWNjyWRjPm9pv0WlIYtLLaOnVsR1RvAdSMgOJPTFNlxS1IkfXxrkmV+gcyQ==" }, + "node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/aggregate-error": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", @@ -1181,14 +1194,6 @@ "node": ">= 4.5.0" } }, - "node_modules/axios": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", - "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", - "dependencies": { - "follow-redirects": "^1.14.0" - } - }, "node_modules/balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1968,7 +1973,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "dependencies": { "ms": "2.1.2" }, @@ -2829,25 +2833,6 @@ "safe-buffer": "~5.1.0" } }, - "node_modules/follow-redirects": { - "version": "1.14.8", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", - "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, "node_modules/for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -3242,6 +3227,18 @@ "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" }, + "node_modules/https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/husky": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", @@ -4418,6 +4415,25 @@ "path-to-regexp": "^1.7.0" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -5244,6 +5260,11 @@ "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=" }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -6558,6 +6579,11 @@ "node": ">=8.0" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", @@ -7056,6 +7082,11 @@ "defaults": "^1.0.3" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, "node_modules/webpack": { "version": "4.46.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz", @@ -7196,6 +7227,15 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -8152,6 +8192,14 @@ "resolved": "https://registry.npmjs.org/afinn-165/-/afinn-165-1.0.3.tgz", "integrity": "sha512-LTwJcUGRzjQCR6uQCXMEUkoecezmWNjyWRjPm9pv0WlIYtLLaOnVsR1RvAdSMgOJPTFNlxS1IkfXxrkmV+gcyQ==" }, + "agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "requires": { + "debug": "^4.3.4" + } + }, "aggregate-error": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.0.1.tgz", @@ -8335,14 +8383,6 @@ "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, - "axios": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.2.tgz", - "integrity": "sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg==", - "requires": { - "follow-redirects": "^1.14.0" - } - }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -9015,7 +9055,6 @@ "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dev": true, "requires": { "ms": "2.1.2" } @@ -9706,11 +9745,6 @@ } } }, - "follow-redirects": { - "version": "1.14.8", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz", - "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==" - }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -10025,6 +10059,15 @@ "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" }, + "https-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz", + "integrity": "sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, "husky": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/husky/-/husky-8.0.3.tgz", @@ -10957,6 +11000,14 @@ "path-to-regexp": "^1.7.0" } }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -11618,6 +11669,11 @@ "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=" }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -12702,6 +12758,11 @@ "is-number": "^7.0.0" } }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "tslib": { "version": "1.14.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", @@ -13127,6 +13188,11 @@ "defaults": "^1.0.3" } }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, "webpack": { "version": "4.46.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz", @@ -13237,6 +13303,15 @@ } } }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index 2aca2c7..e6ca2e4 100644 --- a/package.json +++ b/package.json @@ -51,16 +51,18 @@ }, "dependencies": { "adm-zip": "^0.5.10", - "axios": "^0.21.1", "chalk": "^4.1.0", "commander": "^6.1.0", "fs-extra": "^9.0.1", "glob": "^7.1.6", + "https-proxy-agent": "^7.0.2", "lodash": "^4.17.20", "marked": "^4.0.10", "ms": "^2.1.2", "natural": "^2.1.5", - "ora": "^5.1.0" + "node-fetch": "^2.7.0", + "ora": "^5.1.0", + "proxy-from-env": "^1.1.0" }, "devDependencies": { "eslint": "^8.39.0",