Implement Robust Logging Mechanism for Web Scraping

lib/utils/logger.js

@@ -0,0 +1,127 @@
+/**
+ * Logging utility for web scraping activities
+ * Provides a flexible, configurable logging mechanism
+ */
+class Logger {
+  /**
+   * Creates a new Logger instance
+   * @param {Object} [options={}] - Logging configuration options
+   * @param {string} [options.level='info'] - Logging level
+   * @param {boolean} [options.console=true] - Enable console logging
+   * @param {string} [options.logFile=null] - Path to log file
+   */
+  constructor(options = {}) {
+    this.options = {
+      level: options.level || 'info',
+      console: options.console !== false,
+      logFile: options.logFile || null
+    };
+
+    // Log levels in order of severity
+    this.logLevels = ['debug', 'info', 'warn', 'error'];
+  }
+
+  /**
+   * Check if a log should be output based on current log level
+   * @param {string} level - Log level to check
+   * @returns {boolean} Whether log should be output
+   */
+  _shouldLog(level) {
+    const currentLevelIndex = this.logLevels.indexOf(this.options.level);
+    const messageLevelIndex = this.logLevels.indexOf(level);
+    return messageLevelIndex >= currentLevelIndex;
+  }
+
+  /**
+   * Create a formatted log message
+   * @param {string} level - Log level
+   * @param {string} message - Log message
+   * @param {Object} [metadata={}] - Additional log metadata
+   * @returns {string} Formatted log message
+   */
+  _formatMessage(level, message, metadata = {}) {
+    const timestamp = new Date().toISOString();
+    const metadataStr = Object.keys(metadata).length 
+      ? ` | ${JSON.stringify(metadata)}` 
+      : '';
+    return `[${timestamp}] [${level.toUpperCase()}] ${message}${metadataStr}`;
+  }
+
+  /**
+   * Log a debug message
+   * @param {string} message - Debug message
+   * @param {Object} [metadata={}] - Additional debug metadata
+   */
+  debug(message, metadata = {}) {
+    if (this._shouldLog('debug')) {
+      const formattedMessage = this._formatMessage('debug', message, metadata);
+      if (this.options.console) {
+        console.log(formattedMessage);
+      }
+      this._writeToFile(formattedMessage);
+    }
+  }
+
+  /**
+   * Log an info message
+   * @param {string} message - Info message
+   * @param {Object} [metadata={}] - Additional info metadata
+   */
+  info(message, metadata = {}) {
+    if (this._shouldLog('info')) {
+      const formattedMessage = this._formatMessage('info', message, metadata);
+      if (this.options.console) {
+        console.info(formattedMessage);
+      }
+      this._writeToFile(formattedMessage);
+    }
+  }
+
+  /**
+   * Log a warning message
+   * @param {string} message - Warning message
+   * @param {Object} [metadata={}] - Additional warning metadata
+   */
+  warn(message, metadata = {}) {
+    if (this._shouldLog('warn')) {
+      const formattedMessage = this._formatMessage('warn', message, metadata);
+      if (this.options.console) {
+        console.warn(formattedMessage);
+      }
+      this._writeToFile(formattedMessage);
+    }
+  }
+
+  /**
+   * Log an error message
+   * @param {string} message - Error message
+   * @param {Object} [metadata={}] - Additional error metadata
+   */
+  error(message, metadata = {}) {
+    if (this._shouldLog('error')) {
+      const formattedMessage = this._formatMessage('error', message, metadata);
+      if (this.options.console) {
+        console.error(formattedMessage);
+      }
+      this._writeToFile(formattedMessage);
+    }
+  }
+
+  /**
+   * Write log message to file (if configured)
+   * @param {string} message - Formatted log message
+   * @private
+   */
+  _writeToFile(message) {
+    if (this.options.logFile) {
+      try {
+        const fs = require('fs');
+        fs.appendFileSync(this.options.logFile, message + '\n', 'utf8');
+      } catch (err) {
+        console.error('Failed to write to log file:', err);
+      }
+    }
+  }
+}
+
+module.exports = Logger;

package.json

@@ -25,10 +25,11 @@
   },
   "devDependencies": {
     "jscs": ">=3.0.2",
-    "nodeunit": "0.11.3"
+    "nodeunit": "0.11.3",
+    "mocha": "^10.0.0"
   },
   "scripts": {
-    "test": "node ./node_modules/.bin/nodeunit test"
+    "test": "mocha test/**/*.js"
   },
   "license": "MIT",
   "main": "index",
@@ -39,4 +40,4 @@
   "bugs": {
     "url": "https://github.com/rchipka/node-osmosis/issues"
   }
-}
+}

run_tests.js

@@ -0,0 +1,18 @@
+const Mocha = require('mocha');
+const fs = require('fs');
+const path = require('path');
+
+const mocha = new Mocha();
+
+// Add the test files
+const testDir = path.join(__dirname, 'test');
+fs.readdirSync(testDir)
+  .filter(file => file.endsWith('.js'))
+  .forEach(file => {
+    mocha.addFile(path.join(testDir, file));
+  });
+
+// Run the tests
+mocha.run(function(failures) {
+  process.exitCode = failures ? 1 : 0;
+});

test/logger.js

@@ -0,0 +1,101 @@
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const Logger = require('../lib/utils/logger');
+
+describe('Logger', () => {
+  let consoleLog, consoleInfo, consoleWarn, consoleError;
+  let logFile;
+
+  beforeEach(() => {
+    // Mock console methods
+    consoleLog = console.log;
+    consoleInfo = console.info;
+    consoleWarn = console.warn;
+    consoleError = console.error;
+
+    console.log = console.info = console.warn = console.error = () => {};
+
+    // Create a temp log file
+    logFile = path.join(__dirname, 'test-log.txt');
+  });
+
+  afterEach(() => {
+    // Restore console methods
+    console.log = consoleLog;
+    console.info = consoleInfo;
+    console.warn = consoleWarn;
+    console.error = consoleError;
+
+    // Remove temp log file
+    try {
+      if (fs.existsSync(logFile)) {
+        fs.unlinkSync(logFile);
+      }
+    } catch {}
+  });
+
+  it('should create logger with default options', () => {
+    const logger = new Logger();
+    assert.deepStrictEqual(logger.options, {
+      level: 'info',
+      console: true,
+      logFile: null
+    });
+  });
+
+  it('should log messages at correct levels', () => {
+    const logger = new Logger({ level: 'debug', logFile });
+
+    // Capture logs
+    const logs = [];
+    const originalWrite = logger._writeToFile;
+    logger._writeToFile = (message) => {
+      logs.push(message);
+      originalWrite.call(logger, message);
+    };
+
+    logger.debug('Debug message');
+    logger.info('Info message');
+    logger.warn('Warning message');
+    logger.error('Error message');
+
+    assert.strictEqual(logs.length, 4);
+    assert.ok(logs[0].includes('DEBUG'));
+    assert.ok(logs[1].includes('INFO'));
+    assert.ok(logs[2].includes('WARN'));
+    assert.ok(logs[3].includes('ERROR'));
+  });
+
+  it('should respect log level filtering', () => {
+    const logger = new Logger({ level: 'warn', logFile });
+
+    // Capture logs
+    const logs = [];
+    const originalWrite = logger._writeToFile;
+    logger._writeToFile = (message) => {
+      logs.push(message);
+      originalWrite.call(logger, message);
+    };
+
+    logger.debug('Debug message');
+    logger.info('Info message');
+    logger.warn('Warning message');
+    logger.error('Error message');
+
+    assert.strictEqual(logs.length, 2);
+    assert.ok(logs[0].includes('WARN'));
+    assert.ok(logs[1].includes('ERROR'));
+  });
+
+  it('should format log messages correctly', () => {
+    const logger = new Logger({ logFile });
+    const metadata = { url: 'example.com', status: 200 };
+
+    const message = logger._formatMessage('info', 'Test message', metadata);
+
+    assert.ok(message.includes('[INFO]'));
+    assert.ok(message.includes('Test message'));
+    assert.ok(message.includes(JSON.stringify(metadata)));
+  });
+});