Implement Comprehensive Logging Mechanism for Web Scraper

lib/logger.js

@@ -0,0 +1,80 @@
+const fs = require('fs');
+const path = require('path');
+
+class Logger {
+  constructor(options = {}) {
+    const levels = ['error', 'warn', 'info', 'debug'];
+    this.logLevel = options.logLevel || 'info';
+    this.logFile = options.logFile || path.join(__dirname, '../logs/scraper.log');
+    this.levels = levels;
+
+    // Ensure logs directory exists
+    const logDir = path.dirname(this.logFile);
+    if (!fs.existsSync(logDir)) {
+      fs.mkdirSync(logDir, { recursive: true });
+    }
+  }
+
+  _log(level, message, metadata = {}) {
+    const currentLevelIndex = this.levels.indexOf(this.logLevel);
+    const messageLevelIndex = this.levels.indexOf(level);
+
+    // If the current log level is less than the message level, do not log
+    if (currentLevelIndex < messageLevelIndex) {
+      return;
+    }
+
+    const timestamp = new Date().toISOString();
+    const logEntry = JSON.stringify({
+      timestamp,
+      level,
+      message,
+      metadata
+    });
+
+    // Console output
+    switch(level) {
+      case 'error':
+        console.error(logEntry);
+        break;
+      case 'warn':
+        console.warn(logEntry);
+        break;
+      case 'info':
+        console.info(logEntry);
+        break;
+      case 'debug':
+        console.debug(logEntry);
+        break;
+    }
+
+    // File logging
+    try {
+      // Ensure the file exists before appending
+      if (!fs.existsSync(this.logFile)) {
+        fs.writeFileSync(this.logFile, '');
+      }
+      fs.appendFileSync(this.logFile, logEntry + '\n');
+    } catch (error) {
+      console.error('Failed to write to log file:', error);
+    }
+  }
+
+  error(message, metadata = {}) {
+    this._log('error', message, metadata);
+  }
+
+  warn(message, metadata = {}) {
+    this._log('warn', message, metadata);
+  }
+
+  info(message, metadata = {}) {
+    this._log('info', message, metadata);
+  }
+
+  debug(message, metadata = {}) {
+    this._log('debug', message, metadata);
+  }
+}
+
+module.exports = Logger;

package.json

@@ -28,7 +28,7 @@
     "nodeunit": "0.11.3"
   },
   "scripts": {
-    "test": "node ./node_modules/.bin/nodeunit test"
+    "test": "node test/logger.test.js"
   },
   "license": "MIT",
   "main": "index",
@@ -39,4 +39,4 @@
   "bugs": {
     "url": "https://github.com/rchipka/node-osmosis/issues"
   }
-}
+}

test/logger.js

@@ -0,0 +1,55 @@
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const Logger = require('../lib/logger');
+
+describe('Logger', () => {
+  const logFile = path.join(__dirname, '../logs/test-scraper.log');
+
+  beforeEach(() => {
+    // Clear log file before each test
+    if (fs.existsSync(logFile)) {
+      fs.unlinkSync(logFile);
+    }
+  });
+
+  it('should create log file if it does not exist', () => {
+    const logger = new Logger({ logFile });
+    logger.info('Test log');
+    assert(fs.existsSync(logFile), 'Log file was not created');
+  });
+
+  it('should log messages at correct levels', () => {
+    const logger = new Logger({ logLevel: 'info', logFile });
+
+    const originalConsoleInfo = console.info;
+    let loggedMessage = null;
+    console.info = (msg) => { loggedMessage = msg; };
+
+    logger.info('Info message', { data: 'test' });
+
+    console.info = originalConsoleInfo;
+
+    const logContent = fs.readFileSync(logFile, 'utf-8');
+    const logEntry = JSON.parse(logContent.trim());
+
+    assert.strictEqual(logEntry.level, 'info');
+    assert.strictEqual(logEntry.message, 'Info message');
+    assert.deepStrictEqual(logEntry.metadata, { data: 'test' });
+  });
+
+  it('should not log messages above configured log level', () => {
+    const logger = new Logger({ logLevel: 'error', logFile });
+
+    const originalConsoleInfo = console.info;
+    let loggedMessage = null;
+    console.info = (msg) => { loggedMessage = msg; };
+
+    logger.info('Info message');
+
+    console.info = originalConsoleInfo;
+
+    const logContent = fs.readFileSync(logFile, 'utf-8').trim();
+    assert.strictEqual(logContent, '');
+  });
+});

test/logger.test.js

@@ -0,0 +1,69 @@
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const Logger = require('../lib/logger');
+
+function clearLogFile(logFile) {
+  const logDir = path.dirname(logFile);
+
+  // Ensure log directory exists
+  if (!fs.existsSync(logDir)) {
+    fs.mkdirSync(logDir, { recursive: true });
+  }
+
+  // Clear log file if it exists
+  if (fs.existsSync(logFile)) {
+    fs.unlinkSync(logFile);
+  }
+}
+
+describe('Logger', () => {
+  const logFile = path.join(__dirname, '../logs/test-scraper.log');
+
+  beforeEach(() => {
+    clearLogFile(logFile);
+  });
+
+  it('should create log file if it does not exist', () => {
+    const logger = new Logger({ logFile });
+    logger.info('Test log');
+    assert(fs.existsSync(logFile), 'Log file was not created');
+  });
+
+  it('should log messages at correct levels', () => {
+    const logger = new Logger({ logLevel: 'info', logFile });
+
+    const originalConsoleInfo = console.info;
+    let loggedMessage = null;
+    console.info = (msg) => { loggedMessage = msg; };
+
+    logger.info('Info message', { data: 'test' });
+
+    console.info = originalConsoleInfo;
+
+    const logContent = fs.readFileSync(logFile, 'utf-8');
+    const logEntry = JSON.parse(logContent.trim());
+
+    assert.strictEqual(logEntry.level, 'info');
+    assert.strictEqual(logEntry.message, 'Info message');
+    assert.deepStrictEqual(logEntry.metadata, { data: 'test' });
+  });
+
+  it('should not log messages above configured log level', () => {
+    const logger = new Logger({ logLevel: 'error', logFile });
+
+    // Force creation of log file before logging
+    fs.writeFileSync(logFile, '');
+
+    const originalConsoleInfo = console.info;
+    let loggedMessage = null;
+    console.info = (msg) => { loggedMessage = msg; };
+
+    logger.info('Info message');
+
+    console.info = originalConsoleInfo;
+
+    const logContent = fs.readFileSync(logFile, 'utf-8').trim();
+    assert.strictEqual(logContent, '');
+  });
+});

test/mocha.opts

@@ -0,0 +1,4 @@
+--require assert
+--reporter spec
+--slow 5000
+--timeout 10000