WIP

IFRCGo · Jun 7, 2024 · a31f95e · a31f95e
1 parent 33ab064
commit a31f95e
Show file tree

Hide file tree

Showing 12 changed files with 419 additions and 88 deletions.
diff --git a/app/scripts/translatte/commands/applyMigrations.test.ts b/app/scripts/translatte/commands/applyMigrations.test.ts
@@ -4,7 +4,7 @@ import { join } from 'path';
 
 import { testWithTmpDir } from '../testHelpers';
 import {
-    writeFilePromisify,
+    writeFileAsync,
     readJsonFilesContents,
 } from '../utils';
 import {
@@ -29,7 +29,7 @@ testWithTmpDir('test applyMigrations with no data in server', async ({ tmpdir })
         { name: '000003-1000000000000.json', content: migrationContent3 },
         { name: '000004-1000000000000.json', content: migrationContent4 },
         { name: '000005-1000000000000.json', content: migrationContent5 },
-    ].map(({ name, content }) => writeFilePromisify(
+    ].map(({ name, content }) => writeFileAsync(
         join(tmpdir, 'migrations', name),
         JSON.stringify(content, null, 4),
         'utf8',
@@ -42,7 +42,7 @@ testWithTmpDir('test applyMigrations with no data in server', async ({ tmpdir })
         last_migration: undefined,
         strings: [],
     };
-    await writeFilePromisify(
+    await writeFileAsync(
         join(tmpdir, 'strings', 'before.json'),
         JSON.stringify(emptySourceFile),
         'utf8',
@@ -70,7 +70,7 @@ testWithTmpDir('test applyMigrations with data in server', async ({ tmpdir }) =>
     mkdirSync(join(tmpdir, 'migrations'));
     const migrations = [
         { name: '000006-1000000000000.json', content: migrationContent6 },
-    ].map(({ name, content }) => writeFilePromisify(
+    ].map(({ name, content }) => writeFileAsync(
         join(tmpdir, 'migrations', name),
         JSON.stringify(content, null, 4),
         'utf8',
@@ -79,7 +79,7 @@ testWithTmpDir('test applyMigrations with data in server', async ({ tmpdir }) =>
 
     mkdirSync(join(tmpdir, 'strings'));
 
-    await writeFilePromisify(
+    await writeFileAsync(
         join(tmpdir, 'strings', 'before.json'),
         JSON.stringify(strings1),
         'utf8',

diff --git a/app/scripts/translatte/commands/applyMigrations.ts b/app/scripts/translatte/commands/applyMigrations.ts
@@ -1,11 +1,11 @@
 import { Md5 } from 'ts-md5';
 import { listToMap, isDefined, unique } from '@togglecorp/fujs';
-import { isAbsolute, join, basename } from 'path';
+import { basename } from 'path';
 import {
-    readSource,
-    getMigrationFilesAttrs,
+    readJsonSource,
     readMigrations,
-    writeFilePromisify,
+    writeFileAsync,
+    getMigrationFilesAttrsFromDir,
 } from '../utils';
 import { merge } from './mergeMigrations';
 import {
@@ -39,10 +39,13 @@ function apply(
                 const prevValue = stringsMapping[key];
                 // NOTE: we are comparing hash instead of value so that this works for source language as well as other languages
                 if (prevValue && prevValue.hash !== hash) {
-                    throw `Add: We already have string with different value for namespace '${action.namespace}' and key '${action.key}'`;
+                    // console.info(prevValue, action);
+                    // throw `Add: We already have string with different value for namespace '${action.namespace}' and key '${action.key}'`;
+                    console.info(`Add: We already have string with different value for namespace '${action.namespace}' and key '${action.key}'`);
                 }
 
                 if (newMapping[key]) {
+                    console.info(prevValue, action, newMapping);
                     throw `Add: We already have string for namespace '${action.namespace}' and key '${action.key}' in migration`;
                 }
 
@@ -114,20 +117,16 @@ function apply(
 }
 
 async function applyMigrations(
-    projectPath: string,
-    sourceFileName: string,
+    migrationDir: string,
+    sourceFilePath: string,
     destinationFileName: string,
-    migrationFilePath: string,
     languages: string[],
     from: string | undefined,
     dryRun: boolean | undefined,
 ) {
-    const sourcePath = isAbsolute(sourceFileName)
-        ? sourceFileName
-        : join(projectPath, sourceFileName)
-    const sourceFile = await readSource(sourcePath)
+    const sourceFile = await readJsonSource(sourceFilePath)
 
-    const migrationFilesAttrs = await getMigrationFilesAttrs(projectPath, migrationFilePath);
+    const migrationFilesAttrs = await getMigrationFilesAttrsFromDir(migrationDir);
     const selectedMigrationFilesAttrs = from
         ? migrationFilesAttrs.filter((item) => (item.migrationName > from))
         : migrationFilesAttrs;
@@ -149,25 +148,23 @@ async function applyMigrations(
     );
 
     const outputSourceFileContent: SourceFileContent = {
-        ...sourceFile.content,
+        // ...sourceFile.content,
         last_migration: basename(lastMigration.file),
         strings: apply(
-            sourceFile.content.strings,
+            sourceFile.content.filter(
+                ({ page_name }) => isDefined(page_name)
+            ),
             mergedMigrationActions,
             languages,
         ),
     };
 
-    const destinationPath = isAbsolute(destinationFileName)
-        ? destinationFileName
-        : join(projectPath, destinationFileName)
-
     if (dryRun) {
-        console.info(`Creating file '${destinationPath}'`);
+        console.info(`Creating file '${destinationFileName}'`);
         console.info(outputSourceFileContent);
     } else {
-        await writeFilePromisify(
-            destinationPath,
+        await writeFileAsync(
+            destinationFileName,
             JSON.stringify(outputSourceFileContent, null, 4),
             'utf8',
         );

diff --git a/app/scripts/translatte/commands/exportStrings.ts b/app/scripts/translatte/commands/exportStrings.ts
@@ -0,0 +1,23 @@
+import { join } from 'path';
+
+import { fetchAllServerStrings, writeFileAsync } from "../utils";
+
+async function exportStrings(
+    apiUrl: string,
+    outputDir: string,
+) {
+    const serverStrings = await fetchAllServerStrings(apiUrl);
+
+    const url = new URL(apiUrl);
+    const now = new Date();
+    const exportFileName = `${url.hostname}-${now.getTime()}.json`;
+    const exportFilePath = join(outputDir, exportFileName);
+
+    await writeFileAsync(
+        exportFilePath,
+        JSON.stringify(serverStrings, null, 2),
+        'utf8',
+    );
+}
+
+export default exportStrings;
diff --git a/app/scripts/translatte/commands/generateMigration.ts b/app/scripts/translatte/commands/generateMigration.ts
@@ -2,7 +2,7 @@ import { Md5 } from 'ts-md5';
 import { join, isAbsolute } from 'path';
 
 import {
-    writeFilePromisify,
+    writeFileAsync,
     oneOneMapping,
     readTranslations,
     getTranslationFileNames,
@@ -184,7 +184,7 @@ async function generate(
         console.info(`Creating migration file '${outputMigrationFile}'`);
         console.info(migrationContent);
     } else {
-        await writeFilePromisify(
+        await writeFileAsync(
             outputMigrationFile,
             JSON.stringify(migrationContent, null, 4),
             'utf8',

diff --git a/app/scripts/translatte/commands/importExcel.ts b/app/scripts/translatte/commands/importExcel.ts
@@ -1,12 +1,16 @@
-import { isDefined, isNotDefined, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs';
+import { compareString, isDefined, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToList } from '@togglecorp/fujs';
 import xlsx from 'exceljs';
-import { Language, ServerActionItem } from '../types';
+import { Language, ServerActionItem, SourceStringItem, StringItem } from '../types';
 import { Md5 } from 'ts-md5';
-import { postLanguageStrings } from '../utils';
-
-async function importExcel(importFilePath: string, apiUrl: string, accessToken: string) {
+import { fetchAllServerStrings, postLanguageStrings, readFileAsync, writeFileAsync } from '../utils';
+import stagingStrings from '../../../../../../go-temp/goadmin-stage.ifrc.org-1717130893109.json';
+
+async function importExcel(
+    importFilePath: string,
+    apiUrl: string,
+    accessToken: string,
+) {
     const workbook = new xlsx.Workbook();
-
     await workbook.xlsx.readFile(importFilePath);
 
     const firstSheet = workbook.worksheets[0];
@@ -26,13 +30,7 @@ async function importExcel(importFilePath: string, apiUrl: string, accessToken:
         ({ column }) => column,
     );
 
-    const strings: {
-        key: string;
-        namespace: string;
-        language: Language;
-        value: string;
-        hash: string;
-    }[] = [];
+    const stringsFromExcel: StringItem[] = [];
 
     firstSheet.eachRow(
         (row) => {
@@ -64,38 +62,38 @@ async function importExcel(importFilePath: string, apiUrl: string, accessToken:
 
             const hash = Md5.hashStr(en);
 
-            strings.push({
-                key,
+            stringsFromExcel.push({
                 namespace,
+                key,
                 language: 'en',
                 value: en,
                 hash,
             });
 
             if (isDefined(ar)) {
-                strings.push({
-                    key,
+                stringsFromExcel.push({
                     namespace,
+                    key,
                     language: 'ar',
                     value: ar,
-                hash,
+                    hash,
                 });
             }
 
             if (isDefined(fr)) {
-                strings.push({
-                    key,
+                stringsFromExcel.push({
                     namespace,
+                    key,
                     language: 'fr',
                     value: fr,
                     hash,
                 });
             }
 
             if (isDefined(es)) {
-                strings.push({
-                    key,
+                stringsFromExcel.push({
                     namespace,
+                    key,
                     language: 'es',
                     value: es,
                     hash,
@@ -106,7 +104,7 @@ async function importExcel(importFilePath: string, apiUrl: string, accessToken:
 
     const languageGroupedActions = mapToList(
         listToGroupList(
-            strings,
+            stringsFromExcel,
             ({ language }) => language,
             (languageString) => {
                 const serverAction: ServerActionItem = {
@@ -126,6 +124,7 @@ async function importExcel(importFilePath: string, apiUrl: string, accessToken:
         })
     );
 
+    /*
     const postPromises = languageGroupedActions.map(
         (languageStrings) => postLanguageStrings(
             languageStrings.language,
@@ -135,7 +134,20 @@ async function importExcel(importFilePath: string, apiUrl: string, accessToken:
         )
     )
 
-    await Promise.all(postPromises);
+    const postResponses = await Promise.all(postPromises);
+
+    const postJsonResponses = await Promise.all(
+        postResponses.map((response) => response.json())
+    );
+    */
+
+    /*
+    await writeFileAsync(
+        'serverResponse.json',
+        JSON.stringify(postJsonResponses, null, 2),
+        'utf8',
+    );
+    */
 }
 
 export default importExcel;
diff --git a/app/scripts/translatte/commands/mergeMigrations.ts b/app/scripts/translatte/commands/mergeMigrations.ts
@@ -7,7 +7,7 @@ import {
     getMigrationFilesAttrs,
     readMigrations,
     removeFiles,
-    writeFilePromisify
+    writeFileAsync
 } from '../utils';
 
 function getCanonicalKey(
@@ -209,7 +209,7 @@ async function mergeMigrations(
         console.info(`Creating migration file '${newFileName}'`);
         console.info(mergedMigrationContent);
     } else {
-        await writeFilePromisify(
+        await writeFileAsync(
             newFileName,
             JSON.stringify(mergedMigrationContent, null, 4),
             'utf8',

diff --git a/app/scripts/translatte/commands/pushMigration.ts b/app/scripts/translatte/commands/pushMigration.ts
@@ -1,6 +1,6 @@
-import { isDefined, isNotDefined, listToGroupList, listToMap, mapToMap } from "@togglecorp/fujs";
+import { isDefined, isNotDefined, isTruthyString, listToGroupList, listToMap, mapToMap } from "@togglecorp/fujs";
 import { Language, MigrationActionItem, SourceStringItem } from "../types";
-import { fetchLanguageStrings, getCombinedKey, postLanguageStrings, readMigrations } from "../utils";
+import { fetchLanguageStrings, getCombinedKey, postLanguageStrings, readMigrations, writeFileAsync } from "../utils";
 import { Md5 } from "ts-md5";
 
 const languages: Language[] = ['en', 'fr', 'es', 'ar'];
@@ -38,10 +38,13 @@ async function fetchServerState(apiUrl: string, authToken: string) {
 
 async function pushMigration(migrationFilePath: string, apiUrl: string, authToken: string) {
     const serverStrings = await fetchServerState(apiUrl, authToken);
+    // const serverStrings = prevServerStateStrings as SourceStringItem[];
 
     const serverStringMapByCombinedKey = mapToMap(
         listToGroupList(
-            serverStrings,
+            serverStrings.filter(
+                ({ value, page_name }) => isTruthyString(page_name) && isTruthyString(value),
+            ),
             ({ key, page_name }) => getCombinedKey(key, page_name),
         ),
         (key) => key,
@@ -197,6 +200,38 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke
         }
     });
 
+    await writeFileAsync(
+        'prevServerState.json',
+        JSON.stringify(serverStringMapByCombinedKey, null, 2),
+        'utf8',
+    );
+
+    await writeFileAsync(
+        'serverActions.json',
+        JSON.stringify(serverActions, null, 2),
+        'utf8',
+    );
+
+    /*
+    const postPromise = await postLanguageStrings('en', [
+        {
+            "action": "set",
+            "key": "ifrc",
+            "page_name": "countryPresence",
+            "value": "IFRC",
+            "hash": Md5.hashStr("IFRC"),
+        },
+    ], apiUrl, authToken);
+
+    const postResponseJson = await postPromise.json();
+
+    await writeFilePromisify(
+        'serverResponse.json',
+        JSON.stringify(postResponseJson, null, 2),
+        'utf8',
+    );
+    */
+
     const postResponsePromises = serverActions.map(
         (serverAction) => postLanguageStrings(
             serverAction.language,
@@ -206,7 +241,16 @@ async function pushMigration(migrationFilePath: string, apiUrl: string, authToke
         )
     );
 
-    await Promise.all(postResponsePromises);
+    const postResponses = await Promise.all(postResponsePromises);
+    const postJsonResponses = await Promise.all(
+        postResponses.map((response) => response.json())
+    );
+
+    await writeFileAsync(
+        'serverResponse.json',
+        JSON.stringify(postJsonResponses, null, 2),
+        'utf8',
+    );
 }
 
 export default pushMigration;