allow autoscrape users via your friends list

Signed-off-by: GitHub <[email protected]>

README.md

@@ -82,7 +82,8 @@ Optionally, create a `config.json` file in the root directory of the project. Ot
             "characters": true,
             "games": true,
             "photos": true,
-            "wall": true
+            "wall": true,
+            "yourFriends": true
         },
         "files": {
             "s3": false,

index.ts

@@ -204,7 +204,7 @@ async function main(): Promise<void> {
         statusMessage(MessageType.Critical, 'Users already scraped, skipping user tag scraping...');
     } else {
         statusMessage(MessageType.Info, 'Scraping users...');
-        await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, config.apiKey, config.disabledModules.users, config.manualUserIDs ?? []);
+        await isModuleScraped(database, 'users') ? {} : await getUsers(database, config.domain, sessionID, config.apiKey, config.disabledModules.users, config.manualUserIDs ?? []);
         await insertRow(database, 'scrapers', 'users', true);
         await isModuleScraped(database, 'user_data') ? {} : await getAdditionalUserData(config.domain, sessionID, siteAuth, database, config.disabledModules.users, adminMode);
         await insertRow(database, 'scrapers', 'user_data', true);

package.json

@@ -1,6 +1,6 @@
 {
     "name": "enjinscraper",
-    "version": "1.6.9",
+    "version": "1.7.0",
     "description": "Scrapes an Enjin site via the Enjin API",
     "repository": "https://github.com/Kas-tle/EnjinScraper.git",
     "author": "Joshua Castle <[email protected]",

src/interfaces/friends.ts

@@ -0,0 +1,23 @@
+export namespace Friends {
+    export interface GetList {
+        differential_update: boolean;
+        friends?: Friend[];
+        pages: number;
+        total: number;
+    }
+}
+
+export interface Friend {
+    friend_id: string;
+    is_online: boolean;
+    avatar: string;
+    time: string;
+    seen: string;
+    username: string;
+    sort_username: string;
+    friendship: string;
+    favorite: boolean;
+    fav_position: number | null;
+    network_id: string;
+    quote: string;
+}

src/interfaces/generic.ts

@@ -14,7 +14,7 @@ export interface Params {
     preset_id? : string;
     session_id? : string;
     forum_id? : string;
-    page? : string;
+    page? : string | number;
     thread_id? : string;
     api_key? : string;
     ticket_code? : string;

src/scrapers/users.ts

@@ -11,6 +11,7 @@ import { MessageType, statusMessage } from '../util/console';
 import { Config } from '../util/config';
 import { Profile } from '../interfaces/profile';
 import { getErrorMessage } from '../util/error';
+import { Friends } from '../interfaces/friends';
 
 export async function getAdditionalUserData(domain: string, sessionID: string, siteAuth: SiteAuth | null, database: Database, disabledUserModules: Config["disabledModules"]["users"], adminMode: boolean) {
     statusMessage(MessageType.Info, 'Getting additional user data...')
@@ -383,7 +384,7 @@ export async function getAdditionalUserData(domain: string, sessionID: string, s
     removeExitListeners();
 }
 
-export async function getUsers(database: Database, domain: string, apiKey: string | null, disabledUserModules: Config["disabledModules"]["users"], manualUserIDs: string[]) {
+export async function getUsers(database: Database, domain: string, sessionID: string, apiKey: string | null, disabledUserModules: Config["disabledModules"]["users"], manualUserIDs: string[]) {
     const allUserTags = await getAllUserTags(domain, apiKey, (typeof disabledUserModules === 'object') ? disabledUserModules.tags : false);
     const userDB: UsersDB[] = [];
     const userIDs: string[] = [];
@@ -454,6 +455,9 @@ export async function getUsers(database: Database, domain: string, apiKey: strin
     }
 
     userIDs.push(...manualUserIDs);
+    if (((typeof disabledUserModules === 'object') ? !(disabledUserModules.yourFriends) : true)) {
+        userIDs.push(...await getFriendUserIDs(domain, sessionID));
+    }
     const uniqueUserIDs = [...new Set(userIDs)];
 
     if (uniqueUserIDs.length > 0) {
@@ -495,4 +499,35 @@ async function getColumnUsers(database: Database, table: string, column = 'user_
             }
         });
     });
+}
+
+async function getFriendUserIDs(domain: string, sessionID: string): Promise<string[]> {
+    let totalPages = 1;
+    let page = 1;
+    const allFriends: string[] = [];
+
+    while (page <= totalPages) {
+        const friendsResponse = await enjinRequest<Friends.GetList>({ 
+            session_id: sessionID,
+            type: "all",
+            page
+        }, 'Friends.getList', domain);
+
+        if (friendsResponse.error) {
+            statusMessage(MessageType.Error, `Error getting friends list: ${friendsResponse.error.code} ${friendsResponse.error.message}`)
+            break;
+        }
+
+        const { friends, pages } = friendsResponse.result;
+        totalPages = pages;
+
+        if (friends && friends.length > 0) {
+            allFriends.push(...friends.map(friend => friend.friend_id));
+        }
+        page++;
+    }
+
+    statusMessage(MessageType.Process, `Found ${allFriends.length} friends to be added to scraped users`);
+
+    return allFriends;
 }

src/util/config.ts

@@ -40,6 +40,7 @@ export interface Config {
             games: boolean;
             photos: boolean;
             wall: boolean;
+            yourFriends: boolean;
         };
         files: boolean | {
             s3: boolean;
@@ -98,6 +99,7 @@ const defaultConfig: Config = {
             games: true,
             photos: true,
             wall: true,
+            yourFriends: false,
         },
         files: {
             s3: false,

src/util/notifier.ts

@@ -16,8 +16,8 @@ export async function startNotifier(database: Database, domain: string, apiKey:
     statusMessage(MessageType.Info, 'Press enter to continue or Ctrl + C to exit...');
     await new Promise<void>(resolve => process.stdin.once('data', () => resolve()));
 
-    const disableUserParams = {ips: true, tags: true, fullinfo: true, characters: true, games: true, photos: true, wall: true};
-    await isModuleScraped(database, 'users') ? {} : await getUsers(database, domain, apiKey, disableUserParams, []);
+    const disableUserParams = {ips: true, tags: true, fullinfo: true, characters: true, games: true, photos: true, wall: true, yourFriends: true};
+    await isModuleScraped(database, 'users') ? {} : await getUsers(database, domain, '', apiKey, disableUserParams, []);
 
     const users: { user_id: string, username: string }[] = await new Promise((resolve, reject) => {
         database.all(`SELECT user_id, username FROM users`,