feat: slur replacement using IndexedDB

browser-extension/plugin/package.json

@@ -43,6 +43,7 @@
     "dependencies": {
         "axios": "1.6.2",
         "concurrently": "8.2.2",
+        "dexie": "^4.0.11",
         "dom-to-image": "2.6.0",
         "dom-to-image-improved": "2.8.0",
         "dom-to-image-more": "3.2.0",

browser-extension/plugin/src/api/public-slurs.js

@@ -0,0 +1,24 @@
+import axios from 'axios';
+import config from '../config';
+
+const { API_URL } = config;
+
+export async function getPublicSlurs() {
+    try {
+        const res = await axios.get(`${API_URL}/api/dataset/slur/1`);
+
+        return res.data.slurs;
+    } catch (error) {
+        throw error;
+    }
+}
+
+export async function getPublicSlursMetadata() {
+    try {
+        const res = await axios.get(`${API_URL}/api/dataset/slurmetadata/1`);
+
+        return res.data.slurs_metadata;
+    } catch (error) {
+        throw error;
+    }
+}

browser-extension/plugin/src/background.js

@@ -65,4 +65,4 @@ userBrowserContextMenus.onClicked.addListener((info, tab) => {
         default:
             console('unexpected action');
     }
-});
+});

browser-extension/plugin/src/chrome.js

@@ -70,34 +70,45 @@ const set = (key, value) => {
     });
 };
 
-function sendMessage(type) {
-    if (type == 'updateData') {
-        userBrowserTabs.query(
-            { active: true, currentWindow: true },
-            function (tabs) {
-                userBrowserTabs.sendMessage(
-                    tabs[0].id,
-                    { type: 'updateData' },
-                    function (response) {
-                        console.log(response);
-                    }
-                );
-            }
-        );
-    }
+function sendMessage(type, data = null) {
+    return new Promise((resolve, reject) => {
+        const message = { type };
+        if (data) {
+            message.data = data;
+        }
+
+        if (
+            type === 'updateData' ||
+            type === 'fetchPersonalSlurs' ||
+            type === 'syncApprovedCrowdsourcedSlurs'
+        ) {
+            userBrowserTabs.query(
+                { active: true, currentWindow: true },
+                function (tabs) {
+                    userBrowserTabs.sendMessage(
+                        tabs[0].id,
+                        message,
+                        function (response) {
+                            resolve(response);
+                        }
+                    );
+                }
+            );
+        } else {
+            reject(new Error('Unsupported message type'));
+        }
+    });
 }
 
 function addListener(type, func, response) {
-    chrome.runtime.onMessage.addListener(async function (
-        message,
-        sender,
-        sendResponse
-    ) {
-        if (message.type === type) {
-            func();
-            sendResponse(response);
+    chrome.runtime.onMessage.addListener(
+        async function (message, sender, sendResponse) {
+            if (message.type === type) {
+                func();
+                sendResponse(response);
+            }
         }
-    });
+    );
 }
 
 export default {

browser-extension/plugin/src/content-script.js

@@ -5,10 +5,19 @@ import transform_v2 from './transform-v2';
 import { log } from './logger';
 import repository from './repository';
 const { getUserData, getPreferenceData, setPreferenceData } = repository;
-import { updateSlurList } from './slur-replace';
 import transformGeneral from './transform-general';
 import Api from './ui-components/pages/Api';
+import {
+    initializeSlurs,
+    getSlursBySource,
+    addSlur,
+    deleteSlur,
+    slurExists,
+    bulkAddSlurs,
+    convertSlurMetadataFromDBtoJSON
+} from './slur-store';
 import { createCrowdsourceSlur } from './api/crowdsource-slurs';
+import { getPublicSlurs } from './api/public-slurs';
 
 const { createSlurAndCategory } = Api;
 
@@ -79,85 +88,141 @@ function processPage(newUrl) {
  * eg : When a user clicks on a tweet on their home timeline, they
  * go from the home page to the user status page.
  */
-chrome.runtime.onMessage.addListener(async function (request) {
-    if (request.type === 'updateData') {
-        console.log('data changed. time to update slurs');
-        const preference = await getPreferenceData();
-        console.log(preference);
-        if (preference.slurList != undefined) {
-            updateSlurList(preference.slurList);
-            processPage(location.href);
-        }
-        return true;
-    }
-    if (request.message === 'URL_CHANGED') {
-        const newUrl = request.url;
-        log('Url Changed', newUrl);
-        processPage(location.href);
-        return true;
-    }
-    if (request.type === 'SLUR_ADDED') {
-        const slur = request.slur;
-        log('slur added from bg', slur);
-        const pref = await getPreferenceData();
-        let slurList;
-
-        const user = await getUserData();
-        // console.log('USER in content-script', user);
-        const crowdsourceData = {
-            label: slur,
-        };
-
-        // Adding Slur to Prefrences
-        if (!pref || !pref.slurList) {
-            slurList = slur;
-        } else {
-            slurList = pref.slurList;
-            if (!slurList || slurList === '') {
-                slurList = slur;
-            } else {
-                slurList += `,${slur}`;
+chrome.runtime.onMessage.addListener(
+    async function (request, sender, sendResponse) {
+        if (request.type === 'updateData') {
+            try {
+                const newSlurs = request.data;
+                console.log('New slurs received:', newSlurs);
+                // fetch exisiting slurs
+                const existingSlurs = (await getSlursBySource('personal')).map(
+                    (slur) => slur.word
+                );
+                // Add new slurs to the database
+                for (const slur of newSlurs) {
+                    if (!existingSlurs.includes(slur)) {
+                        await addSlur(slur, 'personal');
+                    }
+                }
+                // Remove slurs from the database that no longer exist in the new list
+                for (const slur of existingSlurs) {
+                    if (!newSlurs.includes(slur)) {
+                        await deleteSlur(slur, 'personal');
+                    }
+                }
+                processPage(location.href);
+            } catch (error) {
+                console.error('Error during updating slur list:', error);
             }
+            return true;
         }
-        try {
-            await setPreferenceData({ ...pref, slurList });
-        } catch (error) {
-            console.error('error updating pref data', error);
+        if (request.type === 'fetchPersonalSlurs') {
+            try {
+                const personalSlurs = await getSlursBySource('personal');
+                const slurArr = personalSlurs.map((slur) => slur.word);
+                console.log('Sending slurs back to pref:', slurArr);
+                return slurArr;
+            } catch (error) {
+                console.error(
+                    'Error fetching personal slurs in content script:',
+                    error
+                );
+                // sendResponse({ success: false, error: error.message });
+            }
         }
+        if (request.type === 'syncApprovedCrowdsourcedSlurs') {
+            const source = 'public_crowdsourced_slurs';
+            try {
+                const publicSlurs = await getPublicSlurs();
+                const publicSlursArray = publicSlurs.map((slur) => slur.label);
+                // console.log(publicSlursArray);
 
-        //Crowdsourcing Slur
-        try {
-            // await createSlurAndCategory(user.accessToken, crowdsourceData);
-            await createCrowdsourceSlur(crowdsourceData, user.token)
-            console.log('finsihed POST req');
-            window.alert(`Slur word "${slur}" added to Uli`);
-        } catch (error) {
-            console.log(error);
+                const filteredSlurs = [];
+                for (const slur of publicSlursArray) {
+                    const exists = await slurExists(slur, source);
+                    if (!exists) {
+                        filteredSlurs.push(slur);
+                    }
+                }
+                // If there are slurs to add, bulk add them to the database
+                if (filteredSlurs.length > 0) {
+                    await bulkAddSlurs(filteredSlurs, source);
+                } else {
+                    console.log('No new slurs to add.');
+                }
+            } catch (error) {
+                console.error('Error fetch public crowsrouced slurs');
+            }
         }
+        if (request.message === 'URL_CHANGED') {
+            const newUrl = request.url;
+            log('Url Changed', newUrl);
+            processPage(location.href);
+            return true;
+        }
+        if (request.type === 'SLUR_ADDED') {
+            const slur = request.slur;
+            log('slur added from bg', slur);
+            let slurList;
 
-        return true;
-    }
+            const user = await getUserData();
+            // console.log('USER in content-script', user);
+            const crowdsourceData = {
+                label: slur
+            };
+
+            // Adding Slur to IndexedDB
+            try {
+                const exists = await slurExists(slur, 'personal');
+
+                if (!exists) {
+                    await addSlur(slur, 'personal');
+                    log('Slur added to IndexedDB:', slur);
+                } else {
+                    log('Slur already exists in IndexedDB, skipping:', slur);
+                }
+            } catch (error) {
+                console.error('Error handling SLUR_ADDED request:', error);
+            }
+
+            //Crowdsourcing Slur
+            try {
+                // await createSlurAndCategory(user.accessToken, crowdsourceData);
+                await createCrowdsourceSlur(crowdsourceData, user.token);
+                console.log('finsihed POST req');
+                window.alert(`Slur word "${slur}" added to Uli`);
+            } catch (error) {
+                console.log(error);
+            }
+
+            return true;
+        }
 
-    if (request.type === 'ULI_ENABLE_SLUR_REPLACEMENT') {
-        console.log('Toggle change event received', request.payload);
-        if (!request.payload) {
-            clearInterval(mainLoadedChecker);
+        if (request.type === 'ULI_ENABLE_SLUR_REPLACEMENT') {
+            console.log('Toggle change event received', request.payload);
+            if (!request.payload) {
+                clearInterval(mainLoadedChecker);
+            }
         }
     }
-});
+);
 
 window.addEventListener(
     'load',
     async () => {
         console.log('content loaded');
         const pref = await getPreferenceData();
-        const { enableSlurReplacement , enableSlurMetadata } = pref;
+        const { enableSlurReplacement, enableSlurMetadata } = pref;
+
+        // Initialize Slurs on content load
+        await initializeSlurs();
+
         if (enableSlurMetadata) {
             let body = document.getElementsByTagName('body');
             let first_body = body[0];
-            transformGeneral.processNewlyAddedNodesGeneral2(first_body);
-        }
-        else if (enableSlurReplacement) {
+            const jsonData = await convertSlurMetadataFromDBtoJSON();
+            transformGeneral.processNewlyAddedNodesGeneral2(first_body, jsonData);
+        } else if (enableSlurReplacement) {
             processPage(location.href);
         }
     },