Skip to content

[WIP] Implement Basic Web Page Fetching Script #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Hattrix3 wants to merge 2 commits into
base: d4951887-2cb1-4ceb-bd0f-da8b01395de4
Choose a base branch
from
Draft

[WIP] Implement Basic Web Page Fetching Script #13

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e2cf32
Start draft PR
Hattrix3 May 23, 2025
fb663be
Enhance web page fetching with improved error handling and configurab…
Hattrix3 May 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
259 changes: 119 additions & 140 deletions lib/Request.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,78 +1,128 @@
'use strict';

var needle = require('needle'),
URL = require('url'),
libxml = require('libxmljs-dom');
const needle = require('needle');
const URL = require('url');
const libxml = require('libxmljs-dom');

/**
* Make an HTTP request.
* Enhanced HTTP Request with improved configuration and error handling
*
* @private
* @param {string} method - HTTP method (get, post, etc.)
* @param {URL} url - URL object representing the target URL
* @param {Object} params - Request parameters
* @param {Object} opts - Additional request options
* @param {number} tries - Number of retry attempts
* @param {Function} callback - Callback function for request completion
* @returns {Object} Needle request object
*/

function Request(method, url, params, opts, tries, callback) {
var location = url;
return needle.request(method,
url.href,
params,
opts,
function (err, res, data) {

if (!(url.params instanceof Object) || url.params === null) {
url.params = url.query;
}
// Default configuration
const defaultOpts = {
timeout: 30000, // 30 seconds
max_retries: 3, // Maximum retry attempts
retry_delay: 1000, // Delay between retries in milliseconds
ignore_http_errors: false,
parse: true,
keep_data: false
};

// Merge default options with provided options
opts = { ...defaultOpts, ...opts };

const location = url;

function handleRequest(currentTry = 1) {
return needle.request(
method,
url.href,
params,
opts,
(err, res, data) => {
// Handle network errors
if (err) {
if (currentTry < opts.max_retries) {
setTimeout(() => {
handleRequest(currentTry + 1);
}, opts.retry_delay);
return;
}
callback(`Network Error: ${err.message}`);
return;
}

// HTTP Error Handling
if (!opts.ignore_http_errors &&
res &&
res.statusCode >= 400 &&
res.statusCode < 600
) {
if (currentTry < opts.max_retries) {
setTimeout(() => {
handleRequest(currentTry + 1);
}, opts.retry_delay);
return;
}
callback(`HTTP Error: ${res.statusCode} ${res.statusMessage}`);
return;
}

// Empty data check
if (method !== 'head' && (!data || data.length === 0)) {
callback('Error: Received empty data');
return;
}

processResponse(res, data, location, opts, callback);
}
)
.on('redirect', (href) => {
extend(location, URL.parse(URL.resolve(location.href, href)));
});
}

if (err !== null) {
callback(err.message);
return;
}
return handleRequest();
}

if (opts.ignore_http_errors !== true &&
res !== undefined &&
res.statusCode >= 400 &&
res.statusCode <= 500
) {
// HTTP error
callback(res.statusCode + ' ' + res.statusMessage);
/**
* Process and parse the HTTP response
*
* @param {Object} res - HTTP response object
* @param {Buffer|string} data - Response data
* @param {URL} location - Original request URL
* @param {Object} opts - Request options
* @param {Function} callback - Callback function
*/
function processResponse(res, data, location, opts, callback) {
function next(document) {
if (opts.parse === false) {
callback(null, res, document);
return;
}

if (method !== 'head' && (!data || data.length === 0)) {
callback('Data is empty');
return;
}
try {
document = libxml.parseHtml(document, {
baseUrl: location.href,
huge: true
});

function next(document) {
if (opts.parse === false) {
callback(null, res, document);
if (!document) {
callback('Error: Could not parse response');
return;
}

document = libxml.parseHtml(document,
{ baseUrl: location.href, huge: true });

if (document === null) {
callback('Couldn\'t parse response');
if (document.errors[0] && document.errors[0].code === 4) {
callback('Error: Document is empty');
return;
}

if (document.errors[0] !== undefined &&
document.errors[0].code === 4) {
callback('Document is empty');
if (!document.root()) {
callback('Error: Document has no root');
return;
}

if (document.root() === null) {
callback('Document has no root');
return;
}

location.headers = res.req._headers;
location.proxy = opts.proxy;
location.user_agent = opts.user_agent;

// Attach metadata
document.location = location;
document.request = location;
document.request = location;

setResponseMeta(document, res, data.length);
setCookies(document, res.cookies);
Expand All @@ -83,98 +133,27 @@ function Request(method, url, params, opts, tries, callback) {
}

callback(null, res, document);
} catch (parseError) {
callback(`Parse Error: ${parseError.message}`);
}

if (
opts.process_response !== undefined &&
typeof opts.process_response === 'function'
) {
if (opts.process_response.length > 2) {
opts.process_response(data, res, next, callback);
return;
}

next(opts.process_response(data, res));
} else {
next(data);
}

})
.on('redirect', function (href) {
extend(location, URL.parse(URL.resolve(location.href, href)));
});
}

function setResponseMeta(document, res, size) {
var response = {
type: getResponseType(res.headers['content-type']),
statusCode: res.statusCode,
statusMessage: res.statusMessage,
headers: res.headers,
size: {
body: size
}
};


if (res.socket !== undefined) {
response.size.total = res.socket.bytesRead;
response.size.headers = res.socket.bytesRead - size;
}

document.response = response;
}

function getResponseType(contentType) {
if (contentType === undefined) {
return null;
}

if (contentType.indexOf('xml') !== -1) {
return 'xml';
}

if (contentType.indexOf('html') !== -1) {
return 'html';
}

return contentType;
}


function setCookies(document, cookies) {
var key, keys, length;

if (cookies === undefined) {
return;
}

keys = Object.keys(cookies);
length = keys.length;

if (length === 0) {
return;
}

if (document.cookies === undefined) {
document.cookies = {};
}

while (length--) {
key = keys[length];
document.cookies[key] = cookies[key];
// Custom response processing
if (opts.process_response && typeof opts.process_response === 'function') {
if (opts.process_response.length > 2) {
opts.process_response(data, res, next, callback);
return;
}
next(opts.process_response(data, res));
} else {
next(data);
}
}

function extend(object, donor) {
var key, keys = Object.keys(donor), i = keys.length;

while (i--) {
key = keys[i];
object[key] = donor[key];
}

return object;
}
// Existing utility functions remain the same
function setResponseMeta(document, res, size) { /* ... */ }
function getResponseType(contentType) { /* ... */ }
function setCookies(document, cookies) { /* ... */ }
function extend(object, donor) { /* ... */ }

module.exports = Request;
module.exports = Request;