diff --git a/README.md b/README.md index 0291d2a..69e4548 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ jsontt try other translation modules on fail | yes, no | default: no -cl, --concurrencylimit optional ↵ | set max concurrency limit (higher faster, but easy to get banned) | default: 3 + -c, --cache optional ↵ | enabled cache | default: no -h, --help display help for command ``` diff --git a/package.json b/package.json index 094b675..5aafae1 100644 --- a/package.json +++ b/package.json @@ -79,16 +79,19 @@ "@iamtraction/google-translate": "^2.0.1", "@types/bluebird": "^3.5.36", "@types/filesystem": "^0.0.32", + "@types/lodash": "^4.17.13", "@vitalets/google-translate-api": "^9.2.0", "axios": "^1.2.2", "bing-translate-api": "^2.8.0", "bluebird": "^3.7.2", "commander": "^10.0.1", + "crypto": "^1.0.1", "cwait": "^1.1.2", "figlet": "^1.6.0", "http-proxy-agent": "^5.0.0", "inquirer": "^7.0.0", "loading-cli": "^1.1.0", + "lodash": "^4.17.21", "openai": "^4.52.3", "yaml": "^2.3.2" } diff --git a/src/cli/cli.ts b/src/cli/cli.ts index 761bd8b..4d42fa5 100644 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -16,7 +16,7 @@ import { translationStatistic, default_concurrency_limit, default_fallback, - fallbacks, + fallbacks, cacheEnableds, } from '../utils/micro'; import { readProxyFile } from '../core/proxy_file'; import { Command, Option, OptionValues } from 'commander'; @@ -27,6 +27,7 @@ import { promptModuleKey, promptFallback, promptConcurrencyLimit, + promptCacheEnabled } from './prompt'; import { TranslationConfig, TranslationModule } from '../modules/modules'; @@ -35,6 +36,7 @@ const program = new Command(); export async function initializeCli() { global.totalTranslation = 0; global.totalTranslated = 0; + global.skipInCache = 0; global.proxyIndex = 0; global.proxyList = []; @@ -62,6 +64,7 @@ export async function initializeCli() { messages.cli.concurrency_limit ) ) + .addOption(new Option(`-c, --cache `, messages.cli.cache_enabled)) .addHelpText( 'after', `\n${messages.cli.usage_with_proxy}\n${messages.cli.usage_by_ops}` @@ -144,6 +147,9 @@ async function translate() { const concurrencyLimitValue = await concurrencyLimit(commandOptions); TranslationConfig.concurrencyLimit = concurrencyLimitValue; + const cacheEnabledValue = await cacheEnabled(commandOptions); + TranslationConfig.cacheEnabled = cacheEnabledValue; + // set loading const { load, refreshInterval } = setLoading(); @@ -158,7 +164,8 @@ async function translate() { load.succeed( `DONE! ${translationStatistic( global.totalTranslation, - global.totalTranslation + global.totalTranslation, + global.skipInCache )}` ); clearInterval(refreshInterval); @@ -191,6 +198,7 @@ async function translationConfig( TranslationModule, concurrencyLimit: default_concurrency_limit, fallback: default_fallback, + cacheEnabled: false, }; return translationConfig; @@ -290,6 +298,28 @@ async function fallback(commandOptions: OptionValues): Promise { return fallback; } +async function cacheEnabled(commandOptions: OptionValues): Promise { + let cacheEnabledStr: string = commandOptions.cacheEnabled ?? undefined; + let cacheEnabled: boolean = false; + + if (!cacheEnabledStr) { + cacheEnabledStr = await promptCacheEnabled(); + + if (!Object.keys(cacheEnableds).includes(cacheEnabledStr)) { + error(`[${cacheEnabledStr}]: ${messages.cli.cache_enabled}`); + process.exit(1); + } + } + + if (cacheEnabledStr === 'yes') { + cacheEnabled = cacheEnableds.yes; + } else { + cacheEnabled = cacheEnableds.no; + } + + return cacheEnabled; +} + async function concurrencyLimit(commandOptions: OptionValues): Promise { let concurrencyLimitInput: number = commandOptions.concurrencylimit ?? undefined; @@ -309,7 +339,8 @@ function setLoading() { const load = loading({ text: `Translating. Please wait. ${translationStatistic( global.totalTranslated, - global.totalTranslation + global.totalTranslation, + global.skipInCache, )}`, color: 'yellow', interval: 100, @@ -320,7 +351,8 @@ function setLoading() { const refreshInterval = setInterval(() => { load.text = `Translating. Please wait. ${translationStatistic( global.totalTranslated, - global.totalTranslation + global.totalTranslation, + global.skipInCache )}`; }, 200); diff --git a/src/cli/prompt.ts b/src/cli/prompt.ts index a3c7923..84f94bb 100644 --- a/src/cli/prompt.ts +++ b/src/cli/prompt.ts @@ -108,6 +108,23 @@ export async function promptFallback() { return answers.fallback; } +export async function promptCacheEnabled() { + const answers = await inquirer.prompt([ + { + type: 'string', + name: 'cache_enabled', + message: messages.cli.cache_enabled, + default: 'no', + }, + ]); + + if (answers.fallback === '') { + return 'no'; + } + + return answers.cache_enabled; +} + export async function promptConcurrencyLimit() { const answers = await inquirer.prompt([ { diff --git a/src/core/core.ts b/src/core/core.ts index 83daff2..9e6f6bc 100644 --- a/src/core/core.ts +++ b/src/core/core.ts @@ -3,6 +3,14 @@ import * as YAML from 'yaml'; import { matchYamlExt } from '../utils/yaml'; import { error, messages } from '../utils/console'; +export async function checkFile(objectPath: string):Promise { + try { + await fs.access(objectPath); + return Promise.resolve(true) + } catch { + return Promise.resolve(false) + } +} export async function getFile(objectPath: string) { let json_file: any = undefined; diff --git a/src/core/json_object.ts b/src/core/json_object.ts index 31c1b0f..6c6992f 100644 --- a/src/core/json_object.ts +++ b/src/core/json_object.ts @@ -1,10 +1,12 @@ import { translatedObject } from '..'; -import { plaintranslate } from './translator'; +import { getKey, plaintranslate} from './translator'; import { TaskQueue } from 'cwait'; import { Promise as bluebirdPromise } from 'bluebird'; import { TranslationConfig } from '../modules/modules'; import { default_concurrency_limit } from '../utils/micro'; import { mergeKeys, removeKeys } from '../utils/console'; +import {checkFile, getFile, saveFilePublic} from "./core"; +import {flatten} from "../utils/json"; var queue = new TaskQueue(bluebirdPromise, default_concurrency_limit); @@ -59,6 +61,17 @@ export async function deepDiver( return null; } + const CACHE_FILE_NAME = `./cache_${from}_${to}.json` + let originalObject:any = JSON.parse(JSON.stringify(object)); + var cacheObject: Record = {}; + if (TranslationConfig.cacheEnabled) { + if (!await checkFile(CACHE_FILE_NAME)) { + await saveFilePublic(CACHE_FILE_NAME, {}); + } + const cacheDataFile = await getFile(CACHE_FILE_NAME); + cacheObject = JSON.parse(cacheDataFile); + } + await Promise.all( Object.keys(object).map(async function (k) { if (has(k)) { @@ -75,7 +88,8 @@ export async function deepDiver( object[k], from, to, - [] + [], + cacheObject ) .then(data => { object[k] = data; @@ -90,5 +104,18 @@ export async function deepDiver( }) ); + if (TranslationConfig.cacheEnabled) { + let originalStructure:Record = flatten(originalObject) + let translatedStructure:Record = flatten(object) + + Object.keys(originalStructure).forEach(key => { + let value = originalStructure[key] + let cacheKey = getKey(value, from, to) + cacheObject[cacheKey] = translatedStructure[key] + }) + + await saveFilePublic(CACHE_FILE_NAME, cacheObject); + } + return object; } diff --git a/src/core/translator.ts b/src/core/translator.ts index 452da5b..b5f6860 100644 --- a/src/core/translator.ts +++ b/src/core/translator.ts @@ -3,17 +3,37 @@ import { getTranslationModuleByKey, translationModuleKeys, } from '../modules/helpers'; -import { TranslationConfig } from '../modules/modules'; +import {TranslationConfig} from '../modules/modules'; import { warn } from '../utils/console'; import { default_value } from '../utils/micro'; import * as ignorer from './ignorer'; +import * as crypto from 'crypto'; + + +export function getKey(str: string, from: string, to: string):string { + let strKey = crypto.createHash('md5').update(str).digest('hex') + return `${from}-${to}-${strKey}`; +} + +export function translateCacheModule(fallbackFn: Function, cacheObject: Record, onSuccess: Function){ + return async (str: string, from: string, to: string): Promise => { + let key = getKey(str, from, to) + if (cacheObject[key] !== undefined && cacheObject[key] !== default_value) { + onSuccess(true) + return Promise.resolve(cacheObject[key]); + } else { + return fallbackFn(str, from, to); + } + }; +} export async function plaintranslate( TranslationConfig: TranslationConfig, str: string, from: string, to: string, - skipModuleKeys: string[] + skipModuleKeys: string[], + cacheObject?: Record ): Promise { // Check for empty strings and return immediately if empty if (str.trim() === '') return str; @@ -28,7 +48,12 @@ export async function plaintranslate( // step: translate in try-catch to keep continuity try { // step: translate with proper source - let translatedStr = await TranslationConfig.TranslationModule.translate( + let defaultTranslateModule: Function = TranslationConfig.TranslationModule.translate + let moduleOrCache = TranslationConfig.cacheEnabled ? translateCacheModule(defaultTranslateModule, cacheObject || {}, ()=> { + global.skipInCache = global.skipInCache + 1; + }) : defaultTranslateModule + + let translatedStr = await moduleOrCache( ignored_str, from, to @@ -46,6 +71,7 @@ export async function plaintranslate( return translatedStr; } catch (e) { // error case + console.log(e) const clonedTranslationConfig = Object.assign({}, TranslationConfig); // cloning to escape ref value const clonedSkipModuleKeys = Object.assign([], skipModuleKeys); // cloning to escape ref value diff --git a/src/global.d.ts b/src/global.d.ts index 4792585..21dd21e 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -3,6 +3,7 @@ export {}; declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyIndex: number; var proxyList: string[]; } diff --git a/src/index.ts b/src/index.ts index 527ad87..60bbe77 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,13 +14,14 @@ const defaults: TranslationConfigTemp = { TranslationModule: TranslationModulesTemp['google'], concurrencyLimit: default_concurrency_limit, fallback: default_fallback, + cacheEnabled: false, }; export async function translateWord( word: string, from: string, to: string, - config: TranslationConfigTemp = defaults + config: TranslationConfigTemp = defaults, ) { return await plaintranslate(config, word, from, to, []); } @@ -32,7 +33,7 @@ export async function translateObject( config: TranslationConfigTemp = defaults ): Promise { let hard_copy = JSON.parse(JSON.stringify(object)); - return objectTranslator(config, hard_copy, from, to); + return objectTranslator(config, hard_copy, from, to, []); } export async function translateFile( diff --git a/src/modules/modules.ts b/src/modules/modules.ts index 1c56903..b5a8039 100644 --- a/src/modules/modules.ts +++ b/src/modules/modules.ts @@ -34,6 +34,7 @@ export type TranslationConfig = { TranslationModule: TranslationModule; concurrencyLimit: number; fallback: boolean; + cacheEnabled: boolean; }; export interface TranslationModule { diff --git a/src/test/core.spec.ts b/src/test/core.spec.ts index ef083cb..66323a7 100644 --- a/src/test/core.spec.ts +++ b/src/test/core.spec.ts @@ -9,6 +9,7 @@ jest.mock('fs/promises'); declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyList: string[]; var proxyIndex: number; } diff --git a/src/test/json-file.spec.ts b/src/test/json-file.spec.ts index 424627a..4ddadde 100644 --- a/src/test/json-file.spec.ts +++ b/src/test/json-file.spec.ts @@ -11,6 +11,7 @@ import { translatedObject } from '..'; declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyList: string[]; var proxyIndex: number; } diff --git a/src/test/json-object.spec.ts b/src/test/json-object.spec.ts index 1ed6a3d..5fb032d 100644 --- a/src/test/json-object.spec.ts +++ b/src/test/json-object.spec.ts @@ -8,6 +8,7 @@ import { default_concurrency_limit, default_fallback } from '../utils/micro'; declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyList: string[]; var proxyIndex: number; } diff --git a/src/test/util.test.ts b/src/test/util.test.ts index d83e9b0..21eeb86 100644 --- a/src/test/util.test.ts +++ b/src/test/util.test.ts @@ -4,6 +4,7 @@ import { error, info, success, warn } from '../utils/console'; declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyList: string[]; var proxyIndex: number; } diff --git a/src/utils/console.ts b/src/utils/console.ts index 6384079..efbe3cc 100644 --- a/src/utils/console.ts +++ b/src/utils/console.ts @@ -53,6 +53,7 @@ export const messages = { from: 'from language | e.g., -f en', to: 'to translates | e.g., -t ar fr zh-CN', new_file_name: 'optional ↵ | output filename | e.g., -n myApp', + cache_enabled: `optional ↵ | enabled cache | yes, no | default: no`, fallback: 'optional ↵ | fallback logic, try other translation modules on fail | yes, no | default: no | e.g., -f yes', concurrency_limit: @@ -81,7 +82,7 @@ export const messages = { proxy_file_notValid_or_not_empty_options: ` - Please ensure that the value for the option "-m, --module " is compatible - Please ensure that the value for the option "-f, --from " is compatible - - nPlease ensure that the value for the option "-t, --to " is compatible + - Please ensure that the value for the option "-t, --to " is compatible - Please ensure that the value for the option "-n, --name " is valid - Please ensure that the value for the option "-f, --fallback " is valid - Please ensure that the value for the option "-cl, --concurrencylimit " is valid diff --git a/src/utils/json.ts b/src/utils/json.ts new file mode 100644 index 0000000..4cdfa56 --- /dev/null +++ b/src/utils/json.ts @@ -0,0 +1,15 @@ +import _ from "lodash"; + +export function flatten (obj: any, parentKey: string = '', separator: string = '.'): Record { + return Object.entries(obj).reduce((acc, [key, value]) => { + const newKey = parentKey ? `${parentKey}${separator}${key}` : key; + + if (_.isObject(value) && !Array.isArray(value)) { + Object.assign(acc, flatten(value, newKey, separator)); + } else { + acc[newKey] = value; + } + + return acc; + }, {} as Record); +}; diff --git a/src/utils/micro.ts b/src/utils/micro.ts index 7beafdd..6c246ec 100644 --- a/src/utils/micro.ts +++ b/src/utils/micro.ts @@ -2,9 +2,10 @@ import * as packageJSON from '../../package.json'; export function translationStatistic( totalTranslated: number, - totalTranslation: number + totalTranslation: number, + skipInCache: number, ): string { - return `${totalTranslated} of ${totalTranslation} translated.`; + return `${totalTranslated} of ${totalTranslation} translated. In cache: ${skipInCache}`; } export function capitalize(str: string): string { return str.charAt(0).toUpperCase() + str.slice(1); @@ -17,4 +18,8 @@ export const fallbacks = { yes: true, no: false, }; +export const cacheEnableds = { + yes: true, + no: false, +}; export const default_fallback = fallbacks.no; diff --git a/test/util.test.ts b/test/util.test.ts index 709cdff..ff50f45 100644 --- a/test/util.test.ts +++ b/test/util.test.ts @@ -4,6 +4,7 @@ import { error, info, success, warn } from '../src/utils/console'; declare global { var totalTranslation: number; var totalTranslated: number; + var skipInCache: number; var proxyList: string[]; var proxyIndex: number; } diff --git a/yarn.lock b/yarn.lock index b0e2cd4..e196c3a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1648,6 +1648,11 @@ dependencies: "@types/node" "*" +"@types/lodash@^4.17.13": + version "4.17.13" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.13.tgz#786e2d67cfd95e32862143abe7463a7f90c300eb" + integrity sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg== + "@types/node-fetch@^2.6.4": version "2.6.11" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24" @@ -2796,6 +2801,11 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +crypto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037" + integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig== + cssom@^0.4.1, cssom@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" @@ -5821,7 +5831,7 @@ lodash.sortby@^4.7.0: resolved "https://registry.yarnpkg.com/lodash.sortby/-/lodash.sortby-4.7.0.tgz#edd14c824e2cc9c1e0b0a1b42bb5210516a42438" integrity sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA== -lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.7.0: +lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.21, lodash@^4.7.0: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==