allow overrideScrapeProgress

Signed-off-by: GitHub <[email protected]>

README.md

@@ -97,7 +97,8 @@ Optionally, create a `config.json` file in the root directory of the project. Ot
     "retrySeconds": 5, // Setting to 0 will retry instantly
     "retryTimes": 5, // Setting to 0 will disable retries; setting to -1 will retry indefinitely
     "debug": true,
-    "disableSSL": false
+    "disableSSL": false,
+    "overrideScrapeProgress": false // setting this to true will ignore the scrapers internal progress tracking and scrape all non-disabled modules
 }
 ```
 

package.json

@@ -1,6 +1,6 @@
 {
     "name": "enjinscraper",
-    "version": "1.7.0",
+    "version": "1.7.1",
     "description": "Scrapes an Enjin site via the Enjin API",
     "repository": "https://github.com/Kas-tle/EnjinScraper.git",
     "author": "Joshua Castle <[email protected]",

src/util/config.ts

@@ -60,6 +60,7 @@ export interface Config {
         messageSubject: string;
         messageBody: string;
     };
+    overrideScrapeProgress?: boolean;
 }
 
 const defaultConfig: Config = {
@@ -113,7 +114,8 @@ const defaultConfig: Config = {
     retrySeconds: 5,
     retryTimes: 5,
     debug: true,
-    disableSSL: false
+    disableSSL: false,
+    overrideScrapeProgress: false
 };
 
 let cachedConfig: Config | null = null;

src/util/database.ts

@@ -1,9 +1,10 @@
 const sqlite3 = require('sqlite3').verbose();
-import path from 'path';
+import path, { resolve } from 'path';
 import { Database } from 'sqlite3';
 import { TableSchema } from '../interfaces/tableschema';
 import { tableSchemas } from './tables';
 import { MessageType, statusMessage } from './console';
+import { getConfig } from './config';
 
 let database: any = null;
 
@@ -261,6 +262,12 @@ export async function updateRows(database: Database, table: string, whereKey: st
 }
 
 export async function isModuleScraped(database: Database, module: string): Promise<boolean> {
+    const config = await getConfig();
+
+    if (config.overrideScrapeProgress === true) {
+        return false;
+    }
+
     return new Promise((resolve, reject) => {
         database.get(`SELECT scraped FROM scrapers WHERE module = ? AND scraped = ?`, [module, true], (err, row) => {
             if (err) {