diff --git a/api/package.json b/api/package.json index af2564bd..06b2a94e 100644 --- a/api/package.json +++ b/api/package.json @@ -59,6 +59,7 @@ "html-to-text": "^9.0.5", "jsonwebtoken": "^9.0.1", "language-tags": "^1.0.8", + "lorem-ipsum": "^2.0.8", "nanoid": "^3.3.0", "node-fetch": "2", "node-html-parser": "^6.1.10", diff --git a/api/src/components/file/file.module.ts b/api/src/components/file/file.module.ts index c3da38d6..bf46c963 100644 --- a/api/src/components/file/file.module.ts +++ b/api/src/components/file/file.module.ts @@ -6,6 +6,6 @@ import { FileRepository } from './file.repository'; @Module({ providers: [FileResolver, FileService, FileRepository], imports: [forwardRef(() => CoreModule)], - exports: [FileService], + exports: [FileService, FileResolver], }) export class FileModule {} diff --git a/api/src/components/populators/populator.module.ts b/api/src/components/populators/populator.module.ts index 4481154f..cd4c5459 100644 --- a/api/src/components/populators/populator.module.ts +++ b/api/src/components/populators/populator.module.ts @@ -4,9 +4,11 @@ import { CoreModule } from 'src/core/core.module'; import { AuthenticationModule } from '../authentication/authentication.module'; import { AuthorizationModule } from '../authorization/authorization.module'; import { DefinitionsModule } from '../definitions/definitions.module'; +import { DocumentsModule } from '../documents/documents.module'; import { FileModule } from '../file/file.module'; import { MapsModule } from '../maps/maps.module'; import { PhraseModule } from '../phrases/phrases.module'; +import { PostModule } from '../post/post.module'; import { BotsModule } from '../translator-bots/bots.module'; import { WordsModule } from '../words/words.module'; import { PopulatorResolver } from './populator.resolver'; @@ -24,6 +26,9 @@ import { PopulatorService } from './populator.service'; DefinitionsModule, WordsModule, PhraseModule, + FileModule, + DocumentsModule, + PostModule, ], providers: [PopulatorResolver, PopulatorService], }) diff --git a/api/src/components/populators/populator.resolver.ts b/api/src/components/populators/populator.resolver.ts index 0af320b0..8cf59a6d 100644 --- a/api/src/components/populators/populator.resolver.ts +++ b/api/src/components/populators/populator.resolver.ts @@ -44,10 +44,6 @@ export class PopulatorResolver { this.pubSub.publish(SubscriptionToken.DataGenerationReport, { [SubscriptionToken.DataGenerationReport]: { output: ``, - mapUploadStatus: SubscriptionStatus.NotStarted, - mapTranslationsStatus: SubscriptionStatus.NotStarted, - mapReTranslationsStatus: SubscriptionStatus.NotStarted, - userCreateStatus: SubscriptionStatus.NotStarted, overallStatus: SubscriptionStatus.NotStarted, } as DataGenProgress, }); @@ -70,8 +66,10 @@ export class PopulatorResolver { input.mapsToLanguages, input.mapAmount, input.userAmount, + input.postsPerUser, input.wordAmount, input.phraseAmount, + input.docAmount, ) .subscribe((n) => this.pubSub.publish(SubscriptionToken.DataGenerationReport, { diff --git a/api/src/components/populators/populator.service.ts b/api/src/components/populators/populator.service.ts index 5c8b16bc..16b4ad99 100644 --- a/api/src/components/populators/populator.service.ts +++ b/api/src/components/populators/populator.service.ts @@ -32,12 +32,20 @@ import { } from './sql-string'; import { DataGenProgress } from './types'; import fetch from 'node-fetch'; +import { FileResolver } from '../file/file.resolver'; +import { DocumentsResolver } from '../documents/documents.resolver'; +import { LoremIpsum } from 'lorem-ipsum'; +import { Readable } from 'stream'; +import { DocumentsService } from '../documents/documents.service'; +import { PostCreateResolver } from '../post/post-create.resolver'; @Injectable() export class PopulatorService { constructor( private httpService: HttpService, private mapRes: MapsResolver, + private fileRes: FileResolver, + private documentsService: DocumentsService, private fileService: FileService, private aiTranslationService: AiTranslationsService, private pg: PostgresService, @@ -46,6 +54,7 @@ export class PopulatorService { private mapVotesService: MapVotesService, private wordVotesService: WordVotesService, private phraseVotesService: PhraseVotesService, + private postCreateResolver: PostCreateResolver, ) {} populateData( @@ -54,8 +63,10 @@ export class PopulatorService { to_languages?: LanguageInput[] | null, mapAmount?: number | null, userAmount?: number | null, + postsPerUser?: number | null, wordAmount?: number | null, phraseAmount?: number | null, + docAmount?: number | null, ) { const value = new BehaviorSubject({ output: ``, @@ -86,7 +97,7 @@ export class PopulatorService { return; } - if (!mapAmount) { + if (!mapAmount && mapAmount != 0) { mapAmount = data.length; } @@ -370,16 +381,39 @@ export class PopulatorService { 'phrase_to_word_translation', ]; - value.next({ - output: `Users voting...`, - overallStatus: SubscriptionStatus.Progressing, - } as DataGenProgress); + // ------------------------------------ + // Voting and Posts + // ------------------------------------ + const userTokens: string[] = []; + const lorem = new LoremIpsum({ + sentencesPerParagraph: { + max: randomInt(5, 50), + min: randomInt(1, 5), + }, + wordsPerSentence: { + max: randomInt(5, 10), + min: randomInt(1, 5), + }, + }); + for (let i = 0; i < passwords.length; i++) { + const { token } = await this.getMockUserToken( + emails[i], + passwords[i], + usernames[i], + ); + userTokens.push(token); + } + translationTableNames.forEach(async (table) => { const tableIdRes = await this.pg.pool.query( `select ${table}_id as id from ${table}s`, ); const tableIds: Array = tableIdRes.rows.map((r) => r.id); for (let i = 0; i < userIds.length; i++) { + value.next({ + output: `User ${usernames[i]}: translation voting`, + overallStatus: SubscriptionStatus.Progressing, + } as DataGenProgress); await this.pg.pool.query( ...callTranslationVoteSetProcedureByTableName({ baseTableName: table, @@ -389,23 +423,34 @@ export class PopulatorService { userId: userIds[i], }), ); + value.next({ + output: `User ${usernames[i]}: translation posting`, + overallStatus: SubscriptionStatus.Progressing, + } as DataGenProgress); + tableIds.forEach(async (id) => { + if (postsPerUser && postsPerUser > 0) { + for (let k = 0; k < postsPerUser; k++) { + await this.postCreateResolver.postCreateResolver( + { + content: + '

' + + lorem.generateSentences(randomInt(1, 20)) + + '

', + parent_id: id, + parent_table: table, + }, + { req: { rawHeaders: ['Bearer ' + tokens[i]] } }, + ); + } + } + }); } }); - value.next({ - output: `${userAmount} users are voting on everything...`, + output: `Users posting and voting on other entities...`, overallStatus: SubscriptionStatus.Progressing, - }); + } as DataGenProgress); - const userTokens: string[] = []; - for (let i = 0; i < passwords.length; i++) { - const { token } = await this.getMockUserToken( - emails[i], - passwords[i], - usernames[i], - ); - userTokens.push(token); - } let res = await this.pg.pool.query( 'select original_map_id as id from original_maps', ); @@ -416,13 +461,28 @@ export class PopulatorService { if (vote == 2) { continue; } - this.mapVotesService.toggleVoteStatus( + await this.mapVotesService.toggleVoteStatus( ids[i], true, vote % 2 === 0 ? true : false, tokens[g], null, ); + if (postsPerUser && postsPerUser > 0) { + for (let k = 0; k < postsPerUser; k++) { + await this.postCreateResolver.postCreateResolver( + { + content: + '

' + + lorem.generateSentences(randomInt(1, 20)) + + '

', + parent_id: ids[i], + parent_table: 'original_maps', + }, + { req: { rawHeaders: ['Bearer ' + tokens[g]] } }, + ); + } + } } } res = await this.pg.pool.query( @@ -442,6 +502,21 @@ export class PopulatorService { tokens[g], null, ); + if (postsPerUser && postsPerUser > 0) { + for (let k = 0; k < postsPerUser; k++) { + await this.postCreateResolver.postCreateResolver( + { + content: + '

' + + lorem.generateSentences(randomInt(1, 20)) + + '

', + parent_id: ids[i], + parent_table: 'translated_maps', + }, + { req: { rawHeaders: ['Bearer ' + tokens[g]] } }, + ); + } + } } } res = await this.pg.pool.query('select word_id as id from words'); @@ -458,6 +533,21 @@ export class PopulatorService { tokens[g], null, ); + if (postsPerUser && postsPerUser > 0) { + for (let k = 0; k < postsPerUser; k++) { + await this.postCreateResolver.postCreateResolver( + { + content: + '

' + + lorem.generateSentences(randomInt(1, 20)) + + '

', + parent_id: ids[i], + parent_table: 'words', + }, + { req: { rawHeaders: ['Bearer ' + tokens[g]] } }, + ); + } + } } } res = await this.pg.pool.query('select phrase_id as id from phrases'); @@ -474,10 +564,78 @@ export class PopulatorService { tokens[g], null, ); + if (postsPerUser && postsPerUser > 0) { + for (let k = 0; k < postsPerUser; k++) { + await this.postCreateResolver.postCreateResolver( + { + content: + '

' + + lorem.generateSentences(randomInt(1, 20)) + + '

', + parent_id: ids[i], + parent_table: 'phrases', + }, + { req: { rawHeaders: ['Bearer ' + tokens[g]] } }, + ); + } + } } } } + // -------------------------------- + // Generate Documents + // -------------------------------- + if (docAmount && docAmount > 0) { + for (let i = 0; i < docAmount; i++) { + // generate lorem ipsum string + value.next({ + output: `Saving document ${i + 1}`, + overallStatus: SubscriptionStatus.Progressing, + } as DataGenProgress); + const lorem = new LoremIpsum({ + sentencesPerParagraph: { + max: randomInt(5, 50), + min: randomInt(1, 5), + }, + wordsPerSentence: { + max: randomInt(5, 10), + min: randomInt(1, 5), + }, + }); + const docContent = lorem.generateParagraphs(randomInt(5, 500)); + const resp = await this.fileService.uploadFile( + Readable.from(docContent), + 'generated' + randomInt(1000), + 'text/plain', + token, + undefined, + ); + if (resp.error != ErrorType.NoError || !resp.file) { + value.next({ + output: `Error - ${resp.error} `, + overallStatus: SubscriptionStatus.Progressing, + } as DataGenProgress); + } + // create document w/ file id + if (resp.file) { + await this.documentsService.saveDocument({ + document: { + file_id: resp.file.id + '', + language_code: 'en', + dialect_code: null, + geo_code: null, + }, + token, + }); + } + } + value.next({ + output: `Documents Saved`, + overallStatus: SubscriptionStatus.Progressing, + } as DataGenProgress); + } + value.next({ output: `Done!`, overallStatus: SubscriptionStatus.Completed, diff --git a/api/src/components/populators/types.ts b/api/src/components/populators/types.ts index db0ed146..a59caa2e 100644 --- a/api/src/components/populators/types.ts +++ b/api/src/components/populators/types.ts @@ -15,6 +15,8 @@ export class DataGenInput { @Field(() => Int, { nullable: true }) userAmount?: number; @Field(() => Int, { nullable: true }) wordAmount?: number; @Field(() => Int, { nullable: true }) phraseAmount?: number; + @Field(() => Int, { nullable: true }) docAmount?: number; + @Field(() => Int, { nullable: true }) postsPerUser?: number; } @ObjectType() diff --git a/api/src/components/translator-bots/ai-translations.service.ts b/api/src/components/translator-bots/ai-translations.service.ts index d729f0fb..bba1fc28 100644 --- a/api/src/components/translator-bots/ai-translations.service.ts +++ b/api/src/components/translator-bots/ai-translations.service.ts @@ -158,7 +158,7 @@ export class AiTranslationsService { ...getTotalWordToPhraseCount( input.fromLanguageCode, input.toLanguageCode, - [google_user_id, lilt_user_id], + [google_user_id, lilt_user_id, gpt_3_user_id, gpt_4_user_id], ), ); translatedMissingWordCount = @@ -181,8 +181,12 @@ export class AiTranslationsService { error: ErrorType.NoError, // later totalPhraseCount, totalWordCount, - translatedMissingPhraseCount, - translatedMissingWordCount, + translatedMissingPhraseCount: + translatedMissingPhraseCount && + (translatedMissingPhraseCount < 0 ? 0 : translatedMissingPhraseCount), + translatedMissingWordCount: + translatedMissingWordCount && + (translatedMissingWordCount < 0 ? 0 : translatedMissingWordCount), googleTranslateTotalLangCount, liltTranslateTotalLangCount, smartcatTranslateTotalLangCount, diff --git a/api/src/schema.gql b/api/src/schema.gql index a82e974d..c7961390 100644 --- a/api/src/schema.gql +++ b/api/src/schema.gql @@ -71,9 +71,11 @@ input CreateQuestionOnWordRangeUpsertInput { } input DataGenInput { + docAmount: Int mapAmount: Int mapsToLanguages: [LanguageInput!] phraseAmount: Int + postsPerUser: Int userAmount: Int wordAmount: Int } diff --git a/api/yarn.lock b/api/yarn.lock index 5551b37a..27698bd8 100644 --- a/api/yarn.lock +++ b/api/yarn.lock @@ -4510,6 +4510,7 @@ __metadata: jsonwebtoken: ^9.0.1 language-tags: ^1.0.8 lint-staged: ^13.2.3 + lorem-ipsum: ^2.0.8 nanoid: ^3.3.0 node-fetch: 2 node-html-parser: ^6.1.10 @@ -5433,6 +5434,13 @@ __metadata: languageName: node linkType: hard +"commander@npm:^9.3.0": + version: 9.5.0 + resolution: "commander@npm:9.5.0" + checksum: c7a3e27aa59e913b54a1bafd366b88650bc41d6651f0cbe258d4ff09d43d6a7394232a4dadd0bf518b3e696fdf595db1028a0d82c785b88bd61f8a440cecfade + languageName: node + linkType: hard + "comment-json@npm:4.2.3": version: 4.2.3 resolution: "comment-json@npm:4.2.3" @@ -8474,6 +8482,17 @@ __metadata: languageName: node linkType: hard +"lorem-ipsum@npm:^2.0.8": + version: 2.0.8 + resolution: "lorem-ipsum@npm:2.0.8" + dependencies: + commander: ^9.3.0 + bin: + lorem-ipsum: dist/bin/lorem-ipsum.bin.js + checksum: 63049aab1f0fa2887fab11dc8a3882f9eef76b2ebc80d0c13659b45709b5c78c291f3328b999f12441ef74f8e49606be62e97033d76e72551629c1467b7b6528 + languageName: node + linkType: hard + "lru-cache@npm:7.10.1 - 7.13.1": version: 7.13.1 resolution: "lru-cache@npm:7.13.1" diff --git a/frontend/graphql.schema.json b/frontend/graphql.schema.json index d9a0b148..1b701e2b 100644 --- a/frontend/graphql.schema.json +++ b/frontend/graphql.schema.json @@ -633,6 +633,18 @@ "description": null, "fields": null, "inputFields": [ + { + "name": "docAmount", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "mapAmount", "description": null, @@ -677,6 +689,18 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "postsPerUser", + "description": null, + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "userAmount", "description": null, diff --git a/frontend/src/components/data-generator/DataGeneratorPage.tsx b/frontend/src/components/data-generator/DataGeneratorPage.tsx index ddda5b2c..4a5006d9 100644 --- a/frontend/src/components/data-generator/DataGeneratorPage.tsx +++ b/frontend/src/components/data-generator/DataGeneratorPage.tsx @@ -19,6 +19,8 @@ export function DataGeneratorPage() { const [mockUserCount, setMockUserCount] = useState(0); const [wordsCount, setWordsCount] = useState(0); const [phrasesCount, setPhrasesCount] = useState(0); + const [docCount, setDocCount] = useState(0); + const [postCount, setPostCount] = useState(0); const [progress, setProgress] = useState(null); const { data } = useSubscribeToDataGenProgressSubscription(); @@ -62,13 +64,17 @@ export function DataGeneratorPage() { userAmount: mockUserCount, phraseAmount: phrasesCount, wordAmount: wordsCount, + docAmount: docCount, + postsPerUser: postCount, }, }); }, [ + docCount, generateData, mapCount, mockUserCount, phrasesCount, + postCount, toLanguageCount, wordsCount, ]); @@ -126,6 +132,20 @@ export function DataGeneratorPage() { onChange={(e) => setMockUserCount(+e.target.value)} /> + + + # Posts per User (will not be triggered unless user count is + supplied) + + setPostCount(+e.target.value)} + /> + # Words setPhrasesCount(+e.target.value)} /> + + # Documents + setDocCount(+e.target.value)} + /> + + {/* Mock map duplications diff --git a/frontend/src/components/data-generator/gql.graphql b/frontend/src/components/data-generator/gql.graphql index 0549182b..ac8b54ed 100644 --- a/frontend/src/components/data-generator/gql.graphql +++ b/frontend/src/components/data-generator/gql.graphql @@ -1,5 +1,5 @@ -mutation GenerateData($mapAmount: Int, $mapsToLanguages: [LanguageInput!], $userAmount: Int, $wordAmount: Int, $phraseAmount: Int) { - generateData(input: {mapAmount: $mapAmount, mapsToLanguages: $mapsToLanguages, userAmount: $userAmount, wordAmount: $wordAmount, phraseAmount: $phraseAmount}) { +mutation GenerateData($mapAmount: Int, $mapsToLanguages: [LanguageInput!], $userAmount: Int, $wordAmount: Int, $phraseAmount: Int, $docAmount: Int, $postsPerUser: Int) { + generateData(input: {mapAmount: $mapAmount, mapsToLanguages: $mapsToLanguages, userAmount: $userAmount, wordAmount: $wordAmount, phraseAmount: $phraseAmount, docAmount: $docAmount, postsPerUser: $postsPerUser}) { error } } diff --git a/frontend/src/components/home/Home.tsx b/frontend/src/components/home/Home.tsx index 3091352d..157c5a82 100644 --- a/frontend/src/components/home/Home.tsx +++ b/frontend/src/components/home/Home.tsx @@ -172,14 +172,14 @@ const Home: React.FC = ({ match }: HomePageProps) => { { link: `/${match.params.nation_id}/${match.params.language_id}/1/translation`, icon: languageOutline, - title: tr('Translation'), + title: tr('Word & Phrase Translation'), description: tr('Translate words and phrases into any language'), isShown: () => !!settings?.isBetaTools, }, { link: `/${match.params.nation_id}/${match.params.language_id}/1/fast-translation`, icon: flashOutline, - title: tr('Fast Translation'), + title: tr('Fast Word & Phrase Translation'), description: tr('Translate words and phrases into any language'), isShown: () => !!settings?.isBetaTools, }, diff --git a/frontend/src/generated/graphql.tsx b/frontend/src/generated/graphql.tsx index 9a6ea8e5..925a9d68 100644 --- a/frontend/src/generated/graphql.tsx +++ b/frontend/src/generated/graphql.tsx @@ -94,9 +94,11 @@ export type CreateQuestionOnWordRangeUpsertInput = { }; export type DataGenInput = { + docAmount?: InputMaybe; mapAmount?: InputMaybe; mapsToLanguages?: InputMaybe>; phraseAmount?: InputMaybe; + postsPerUser?: InputMaybe; userAmount?: InputMaybe; wordAmount?: InputMaybe; }; @@ -3098,6 +3100,8 @@ export type GenerateDataMutationVariables = Exact<{ userAmount?: InputMaybe; wordAmount?: InputMaybe; phraseAmount?: InputMaybe; + docAmount?: InputMaybe; + postsPerUser?: InputMaybe; }>; @@ -5402,9 +5406,9 @@ export type PasswordResetFormRequestMutationHookResult = ReturnType; export type PasswordResetFormRequestMutationOptions = Apollo.BaseMutationOptions; export const GenerateDataDocument = gql` - mutation GenerateData($mapAmount: Int, $mapsToLanguages: [LanguageInput!], $userAmount: Int, $wordAmount: Int, $phraseAmount: Int) { + mutation GenerateData($mapAmount: Int, $mapsToLanguages: [LanguageInput!], $userAmount: Int, $wordAmount: Int, $phraseAmount: Int, $docAmount: Int, $postsPerUser: Int) { generateData( - input: {mapAmount: $mapAmount, mapsToLanguages: $mapsToLanguages, userAmount: $userAmount, wordAmount: $wordAmount, phraseAmount: $phraseAmount} + input: {mapAmount: $mapAmount, mapsToLanguages: $mapsToLanguages, userAmount: $userAmount, wordAmount: $wordAmount, phraseAmount: $phraseAmount, docAmount: $docAmount, postsPerUser: $postsPerUser} ) { error } @@ -5430,6 +5434,8 @@ export type GenerateDataMutationFn = Apollo.MutationFunction