Add caching mechanism to translation process

README.md

@@ -113,6 +113,7 @@ jsontt <your/path/to/file.yaml/yml>
                                     try other translation modules on fail | yes, no | default: no
   -cl, --concurrencylimit <number>  optional ↵ | set max concurrency limit
                                     (higher faster, but easy to get banned) | default: 3
+  -c, --cache                       optional ↵ | enabled cache | default: no
   -h, --help                        display help for command
 ```
 

package.json

@@ -79,16 +79,19 @@
     "@iamtraction/google-translate": "^2.0.1",
     "@types/bluebird": "^3.5.36",
     "@types/filesystem": "^0.0.32",
+    "@types/lodash": "^4.17.13",
     "@vitalets/google-translate-api": "^9.2.0",
     "axios": "^1.2.2",
     "bing-translate-api": "^2.8.0",
     "bluebird": "^3.7.2",
     "commander": "^10.0.1",
+    "crypto": "^1.0.1",
     "cwait": "^1.1.2",
     "figlet": "^1.6.0",
     "http-proxy-agent": "^5.0.0",
     "inquirer": "^7.0.0",
     "loading-cli": "^1.1.0",
+    "lodash": "^4.17.21",
     "openai": "^4.52.3",
     "yaml": "^2.3.2"
   }

src/cli/cli.ts

@@ -16,7 +16,7 @@ import {
   translationStatistic,
   default_concurrency_limit,
   default_fallback,
-  fallbacks,
+  fallbacks, cacheEnableds,
 } from '../utils/micro';
 import { readProxyFile } from '../core/proxy_file';
 import { Command, Option, OptionValues } from 'commander';
@@ -27,6 +27,7 @@ import {
   promptModuleKey,
   promptFallback,
   promptConcurrencyLimit,
+  promptCacheEnabled
 } from './prompt';
 import { TranslationConfig, TranslationModule } from '../modules/modules';
 
@@ -35,6 +36,7 @@ const program = new Command();
 export async function initializeCli() {
   global.totalTranslation = 0;
   global.totalTranslated = 0;
+  global.skipInCache = 0;
   global.proxyIndex = 0;
   global.proxyList = [];
 
@@ -62,6 +64,7 @@ export async function initializeCli() {
         messages.cli.concurrency_limit
       )
     )
+    .addOption(new Option(`-c, --cache <boolean>`, messages.cli.cache_enabled))
     .addHelpText(
       'after',
       `\n${messages.cli.usage_with_proxy}\n${messages.cli.usage_by_ops}`
@@ -144,6 +147,9 @@ async function translate() {
   const concurrencyLimitValue = await concurrencyLimit(commandOptions);
   TranslationConfig.concurrencyLimit = concurrencyLimitValue;
 
+  const cacheEnabledValue = await cacheEnabled(commandOptions);
+  TranslationConfig.cacheEnabled = cacheEnabledValue;
+
   // set loading
   const { load, refreshInterval } = setLoading();
 
@@ -158,7 +164,8 @@ async function translate() {
   load.succeed(
     `DONE! ${translationStatistic(
       global.totalTranslation,
-      global.totalTranslation
+      global.totalTranslation,
+      global.skipInCache
     )}`
   );
   clearInterval(refreshInterval);
@@ -191,6 +198,7 @@ async function translationConfig(
     TranslationModule,
     concurrencyLimit: default_concurrency_limit,
     fallback: default_fallback,
+    cacheEnabled: false,
   };
 
   return translationConfig;
@@ -290,6 +298,28 @@ async function fallback(commandOptions: OptionValues): Promise<boolean> {
   return fallback;
 }
 
+async function cacheEnabled(commandOptions: OptionValues): Promise<boolean> {
+  let cacheEnabledStr: string = commandOptions.cacheEnabled ?? undefined;
+  let cacheEnabled: boolean = false;
+
+  if (!cacheEnabledStr) {
+    cacheEnabledStr = await promptCacheEnabled();
+
+    if (!Object.keys(cacheEnableds).includes(cacheEnabledStr)) {
+      error(`[${cacheEnabledStr}]: ${messages.cli.cache_enabled}`);
+      process.exit(1);
+    }
+  }
+
+  if (cacheEnabledStr === 'yes') {
+    cacheEnabled = cacheEnableds.yes;
+  } else {
+    cacheEnabled = cacheEnableds.no;
+  }
+
+  return cacheEnabled;
+}
+
 async function concurrencyLimit(commandOptions: OptionValues): Promise<number> {
   let concurrencyLimitInput: number =
     commandOptions.concurrencylimit ?? undefined;
@@ -309,7 +339,8 @@ function setLoading() {
   const load = loading({
     text: `Translating. Please wait. ${translationStatistic(
       global.totalTranslated,
-      global.totalTranslation
+      global.totalTranslation,
+      global.skipInCache,
     )}`,
     color: 'yellow',
     interval: 100,
@@ -320,7 +351,8 @@ function setLoading() {
   const refreshInterval = setInterval(() => {
     load.text = `Translating. Please wait. ${translationStatistic(
       global.totalTranslated,
-      global.totalTranslation
+      global.totalTranslation,
+      global.skipInCache
     )}`;
   }, 200);
 

src/cli/prompt.ts

@@ -108,6 +108,23 @@ export async function promptFallback() {
   return answers.fallback;
 }
 
+export async function promptCacheEnabled() {
+  const answers = await inquirer.prompt([
+    {
+      type: 'string',
+      name: 'cache_enabled',
+      message: messages.cli.cache_enabled,
+      default: 'no',
+    },
+  ]);
+
+  if (answers.fallback === '') {
+    return 'no';
+  }
+
+  return answers.cache_enabled;
+}
+
 export async function promptConcurrencyLimit() {
   const answers = await inquirer.prompt([
     {

src/core/core.ts

@@ -3,6 +3,14 @@ import * as YAML from 'yaml';
 import { matchYamlExt } from '../utils/yaml';
 import { error, messages } from '../utils/console';
 
+export async function checkFile(objectPath: string):Promise<boolean> {
+  try {
+    await fs.access(objectPath);
+    return Promise.resolve(true)
+  } catch {
+    return Promise.resolve(false)
+  }
+}
 export async function getFile(objectPath: string) {
   let json_file: any = undefined;
 

src/core/json_object.ts

@@ -1,10 +1,12 @@
 import { translatedObject } from '..';
-import { plaintranslate } from './translator';
+import { getKey, plaintranslate} from './translator';
 import { TaskQueue } from 'cwait';
 import { Promise as bluebirdPromise } from 'bluebird';
 import { TranslationConfig } from '../modules/modules';
 import { default_concurrency_limit } from '../utils/micro';
 import { mergeKeys, removeKeys } from '../utils/console';
+import {checkFile, getFile, saveFilePublic} from "./core";
+import {flatten} from "../utils/json";
 
 var queue = new TaskQueue(bluebirdPromise, default_concurrency_limit);
 
@@ -59,6 +61,17 @@ export async function deepDiver(
     return null;
   }
 
+  const CACHE_FILE_NAME = `./cache_${from}_${to}.json`
+  let originalObject:any = JSON.parse(JSON.stringify(object));
+  var cacheObject: Record<string, any> = {};
+  if (TranslationConfig.cacheEnabled) {
+    if (!await checkFile(CACHE_FILE_NAME)) {
+      await saveFilePublic(CACHE_FILE_NAME, {});
+    }
+    const cacheDataFile = await getFile(CACHE_FILE_NAME);
+    cacheObject = JSON.parse(cacheDataFile);
+  }
+
   await Promise.all(
     Object.keys(object).map(async function (k) {
       if (has(k)) {
@@ -75,7 +88,8 @@ export async function deepDiver(
                 object[k],
                 from,
                 to,
-                []
+                [],
+                cacheObject
               )
                 .then(data => {
                   object[k] = data;
@@ -90,5 +104,18 @@ export async function deepDiver(
     })
   );
 
+  if (TranslationConfig.cacheEnabled) {
+    let originalStructure:Record<string, string> = flatten(originalObject)
+    let translatedStructure:Record<string, string> = flatten(object)
+
+    Object.keys(originalStructure).forEach(key => {
+      let value = originalStructure[key]
+      let cacheKey = getKey(value, from, to)
+      cacheObject[cacheKey] = translatedStructure[key]
+    })
+
+    await saveFilePublic(CACHE_FILE_NAME, cacheObject);
+  }
+
   return object;
 }

src/core/translator.ts

@@ -3,17 +3,37 @@ import {
   getTranslationModuleByKey,
   translationModuleKeys,
 } from '../modules/helpers';
-import { TranslationConfig } from '../modules/modules';
+import {TranslationConfig} from '../modules/modules';
 import { warn } from '../utils/console';
 import { default_value } from '../utils/micro';
 import * as ignorer from './ignorer';
+import * as crypto from 'crypto';
+
+
+export function getKey(str: string, from: string, to: string):string {
+  let strKey = crypto.createHash('md5').update(str).digest('hex')
+  return `${from}-${to}-${strKey}`;
+}
+
+export function translateCacheModule(fallbackFn: Function, cacheObject: Record<string, any>, onSuccess: Function){
+  return async (str: string, from: string, to: string): Promise<string> => {
+    let key =  getKey(str, from, to)
+    if (cacheObject[key] !== undefined && cacheObject[key] !== default_value) {
+      onSuccess(true)
+      return Promise.resolve(cacheObject[key]);
+    } else {
+      return fallbackFn(str, from, to);
+    }
+  };
+}
 
 export async function plaintranslate(
   TranslationConfig: TranslationConfig,
   str: string,
   from: string,
   to: string,
-  skipModuleKeys: string[]
+  skipModuleKeys: string[],
+  cacheObject?: Record<string, any>
 ): Promise<string> {
   // Check for empty strings and return immediately if empty
   if (str.trim() === '') return str;
@@ -28,7 +48,12 @@ export async function plaintranslate(
   // step: translate in try-catch to keep continuity
   try {
     // step: translate with proper source
-    let translatedStr = await TranslationConfig.TranslationModule.translate(
+    let defaultTranslateModule: Function = TranslationConfig.TranslationModule.translate
+    let moduleOrCache = TranslationConfig.cacheEnabled ? translateCacheModule(defaultTranslateModule, cacheObject || {}, ()=> {
+      global.skipInCache = global.skipInCache + 1;
+    }) : defaultTranslateModule
+
+    let translatedStr = await moduleOrCache(
       ignored_str,
       from,
       to
@@ -46,6 +71,7 @@ export async function plaintranslate(
     return translatedStr;
   } catch (e) {
     // error case
+    console.log(e)
     const clonedTranslationConfig = Object.assign({}, TranslationConfig); // cloning to escape ref value
     const clonedSkipModuleKeys = Object.assign([], skipModuleKeys); // cloning to escape ref value
 

src/global.d.ts

@@ -3,6 +3,7 @@ export {};
 declare global {
   var totalTranslation: number;
   var totalTranslated: number;
+  var skipInCache: number;
   var proxyIndex: number;
   var proxyList: string[];
 }

src/index.ts

@@ -14,13 +14,14 @@ const defaults: TranslationConfigTemp = {
   TranslationModule: TranslationModulesTemp['google'],
   concurrencyLimit: default_concurrency_limit,
   fallback: default_fallback,
+  cacheEnabled: false,
 };
 
 export async function translateWord(
   word: string,
   from: string,
   to: string,
-  config: TranslationConfigTemp = defaults
+  config: TranslationConfigTemp = defaults,
 ) {
   return await plaintranslate(config, word, from, to, []);
 }
@@ -32,7 +33,7 @@ export async function translateObject(
   config: TranslationConfigTemp = defaults
 ): Promise<translatedObject | translatedObject[]> {
   let hard_copy = JSON.parse(JSON.stringify(object));
-  return objectTranslator(config, hard_copy, from, to);
+  return objectTranslator(config, hard_copy, from, to, []);
 }
 
 export async function translateFile(

src/modules/modules.ts

@@ -34,6 +34,7 @@ export type TranslationConfig = {
   TranslationModule: TranslationModule;
   concurrencyLimit: number;
   fallback: boolean;
+  cacheEnabled: boolean;
 };
 
 export interface TranslationModule {

src/test/core.spec.ts

@@ -9,6 +9,7 @@ jest.mock('fs/promises');
 declare global {
   var totalTranslation: number;
   var totalTranslated: number;
+  var skipInCache: number;
   var proxyList: string[];
   var proxyIndex: number;
 }

src/test/json-file.spec.ts

@@ -11,6 +11,7 @@ import { translatedObject } from '..';
 declare global {
   var totalTranslation: number;
   var totalTranslated: number;
+  var skipInCache: number;
   var proxyList: string[];
   var proxyIndex: number;
 }

src/test/json-object.spec.ts

@@ -8,6 +8,7 @@ import { default_concurrency_limit, default_fallback } from '../utils/micro';
 declare global {
   var totalTranslation: number;
   var totalTranslated: number;
+  var skipInCache: number;
   var proxyList: string[];
   var proxyIndex: number;
 }

src/test/util.test.ts

@@ -4,6 +4,7 @@ import { error, info, success, warn } from '../utils/console';
 declare global {
   var totalTranslation: number;
   var totalTranslated: number;
+  var skipInCache: number;
   var proxyList: string[];
   var proxyIndex: number;
 }

src/utils/console.ts

@@ -53,6 +53,7 @@ export const messages = {
     from: 'from language | e.g., -f en',
     to: 'to translates | e.g., -t ar fr zh-CN',
     new_file_name: 'optional ↵ | output filename | e.g., -n myApp',
+    cache_enabled: `optional ↵ | enabled cache | yes, no | default: no`,
     fallback:
       'optional ↵ | fallback logic, try other translation modules on fail | yes, no | default: no | e.g., -f yes',
     concurrency_limit:
@@ -81,7 +82,7 @@ export const messages = {
     proxy_file_notValid_or_not_empty_options: `
     - Please ensure that the value for the option "-m, --module <Translation>" is compatible
     - Please ensure that the value for the option "-f, --from <Language>" is compatible
-    - nPlease ensure that the value for the option "-t, --to <Languages...>" is compatible
+    - Please ensure that the value for the option "-t, --to <Languages...>" is compatible
     - Please ensure that the value for the option "-n, --name <string>" is valid
     - Please ensure that the value for the option "-f, --fallback <string>" is valid
     - Please ensure that the value for the option "-cl, --concurrencylimit <number>" is valid