diff --git a/web/app/feature/translate/functions/mutations.server.ts b/web/app/feature/translate/functions/mutations.server.ts deleted file mode 100644 index 9e60fd13..00000000 --- a/web/app/feature/translate/functions/mutations.server.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { createHash } from "node:crypto"; -import { prisma } from "~/utils/prisma"; - -export async function getOrCreateSourceTextIdAndPageVersionSourceText( - text: string, - number: number, - pageVersionId: number, -): Promise { - const textHash = Buffer.from( - createHash("sha256").update(text).digest("hex"), - "hex", - ); - - return prisma.$transaction(async (tx) => { - const sourceText = await tx.sourceText.upsert({ - where: { - textHash_number: { - textHash, - number, - }, - }, - update: {}, - create: { - text, - number, - textHash, - }, - }); - - await tx.pageVersionSourceText.upsert({ - where: { - pageVersionId_sourceTextId: { - pageVersionId, - sourceTextId: sourceText.id, - }, - }, - update: {}, - create: { - pageVersionId, - sourceTextId: sourceText.id, - }, - }); - - return sourceText.id; - }); -} diff --git a/web/app/feature/translate/libs/translation.ts b/web/app/feature/translate/libs/translation.ts deleted file mode 100644 index 27527ebb..00000000 --- a/web/app/feature/translate/libs/translation.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { getOrCreatePageId } from "../../../libs/pageService"; -import { getOrCreatePageVersionId } from "../../../libs/pageVersion"; -import { - getOrCreateUserAITranslationInfo, - updateUserAITranslationInfo, -} from "../../../libs/userAITranslationInfo"; -import type { NumberedElement } from "../../../routes/translate/types"; -import { - getOrCreateTranslations, - splitNumberedElements, -} from "./translationUtils"; - -export async function translate( - geminiApiKey: string, - aiModel: string, - userId: number, - targetLanguage: string, - title: string, - numberedContent: string, - numberedElements: NumberedElement[], - url: string, -): Promise { - const pageId = await getOrCreatePageId(url || ""); - const pageVersionId = await getOrCreatePageVersionId( - url, - title, - numberedContent, - pageId, - ); - - const userAITranslationInfo = await getOrCreateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - ); - - if (userAITranslationInfo.aiTranslationStatus === "completed") { - return userAITranslationInfo.aiTranslationStatus; - } - - await processTranslation( - geminiApiKey, - aiModel, - userId, - pageId, - pageVersionId, - targetLanguage, - title, - numberedElements, - ); - - return userAITranslationInfo.aiTranslationStatus; -} - -export async function processTranslation( - geminiApiKey: string, - aiModel: string, - userId: number, - pageId: number, - pageVersionId: number, - targetLanguage: string, - title: string, - numberedElements: NumberedElement[], -) { - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "in_progress", - 0, - ); - try { - const chunks = splitNumberedElements(numberedElements); - const totalChunks = chunks.length; - for (let i = 0; i < chunks.length; i++) { - console.log(`Processing chunk ${i + 1} of ${totalChunks}`); - - await getOrCreateTranslations( - geminiApiKey, - aiModel, - chunks[i], - targetLanguage, - pageId, - pageVersionId, - title, - ); - const progress = ((i + 1) / totalChunks) * 100; - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "in_progress", - progress, - ); - } - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "completed", - 100, - ); - } catch (error) { - console.error("Background translation job failed:", error); - await updateUserAITranslationInfo( - userId, - pageVersionId, - targetLanguage, - "failed", - 0, - ); - } -} diff --git a/web/app/feature/translate/libs/translationUtils.ts b/web/app/feature/translate/libs/translationUtils.ts deleted file mode 100644 index 24b03b51..00000000 --- a/web/app/feature/translate/libs/translationUtils.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { getOrCreateAIUser } from "~/libs/db/user.server"; -import { getOrCreatePageVersionTranslationInfo } from "../../../libs/pageVersionTranslationInfo"; -import { MAX_CHUNK_SIZE } from "../../../routes/translate/constants"; -import type { NumberedElement } from "../../../routes/translate/types"; -import { prisma } from "../../../utils/prisma"; -import { getOrCreateSourceTextIdAndPageVersionSourceText } from "../functions/mutations.server"; -import { getGeminiModelResponse } from "../utils/gemini"; - -export function splitNumberedElements( - elements: NumberedElement[], -): NumberedElement[][] { - const chunks: NumberedElement[][] = []; - let currentChunk: NumberedElement[] = []; - let currentSize = 0; - - for (const element of elements) { - if ( - currentSize + element.text.length > MAX_CHUNK_SIZE && - currentChunk.length > 0 - ) { - chunks.push(currentChunk); - currentChunk = []; - currentSize = 0; - } - currentChunk.push(element); - currentSize += element.text.length; - } - - if (currentChunk.length > 0) { - chunks.push(currentChunk); - } - return chunks; -} - -export function extractTranslations( - text: string, -): { number: number; text: string }[] { - try { - const parsed = JSON.parse(text); - if (Array.isArray(parsed)) { - return parsed; - } - } catch (error) { - console.warn("Failed to parse JSON, falling back to regex parsing", error); - } - - const translations: { number: number; text: string }[] = []; - const regex = - /{\s*"number"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"((?:\\.|[^"\\])*)"\s*}/g; - let match: RegExpExecArray | null; - - while (true) { - match = regex.exec(text); - if (match === null) break; - - translations.push({ - number: Number.parseInt(match[1], 10), - text: match[2], - }); - } - - return translations; -} - -export async function getOrCreateTranslations( - geminiApiKey: string, - aiModel: string, - elements: NumberedElement[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise { - const translations: NumberedElement[] = []; - const untranslatedElements: NumberedElement[] = []; - const sourceTextsId = await Promise.all( - elements.map((element) => - getOrCreateSourceTextIdAndPageVersionSourceText( - element.text, - element.number, - pageVersionId, - ), - ), - ); - - const existingTranslations = await prisma.translateText.findMany({ - where: { - sourceTextId: { in: sourceTextsId }, - targetLanguage, - }, - orderBy: [{ point: "desc" }, { createdAt: "desc" }], - }); - - const translationMap = new Map( - existingTranslations.map((t) => [t.sourceTextId, t]), - ); - - elements.forEach((element, index) => { - const sourceTextId = sourceTextsId[index]; - const existingTranslation = translationMap.get(sourceTextId); - - if (existingTranslation) { - translations.push({ - number: element.number, - text: existingTranslation.text, - }); - } else { - untranslatedElements.push(element); - } - }); - - if (untranslatedElements.length > 0) { - const newTranslations = await translateUntranslatedElements( - geminiApiKey, - aiModel, - untranslatedElements, - targetLanguage, - pageId, - pageVersionId, - title, - ); - translations.push(...newTranslations); - } - - return translations.sort((a, b) => a.number - b.number); -} - -async function translateUntranslatedElements( - geminiApiKey: string, - aiModel: string, - untranslatedElements: NumberedElement[], - targetLanguage: string, - pageId: number, - pageVersionId: number, - title: string, -): Promise { - const source_text = untranslatedElements - .map((el) => JSON.stringify(el)) - .join("\n"); - const translatedText = await getGeminiModelResponse( - geminiApiKey, - aiModel, - title, - source_text, - targetLanguage, - ); - - const extractedTranslations = extractTranslations(translatedText); - await getOrCreatePageVersionTranslationInfo( - pageVersionId, - targetLanguage, - extractedTranslations[0].text, - ); - - const systemUserId = await getOrCreateAIUser(aiModel); - - await Promise.all( - extractedTranslations.map(async (translation) => { - const sourceText = untranslatedElements.find( - (el) => el.number === translation.number, - )?.text; - - if (!sourceText) { - console.error( - `Source text not found for translation number ${translation.number}`, - ); - return; - } - - const sourceTextId = - await getOrCreateSourceTextIdAndPageVersionSourceText( - sourceText, - translation.number, - pageVersionId, - ); - await prisma.translateText.create({ - data: { - targetLanguage, - text: translation.text, - sourceTextId, - userId: systemUserId, - }, - }); - }), - ); - - return extractedTranslations; -} diff --git a/web/app/feature/translate/utils/addNumbersToContent.ts b/web/app/features/prepare-html-for-translate/utils/addNumbersToContent.ts similarity index 100% rename from web/app/feature/translate/utils/addNumbersToContent.ts rename to web/app/features/prepare-html-for-translate/utils/addNumbersToContent.ts diff --git a/web/app/feature/translate/utils/extractArticle.ts b/web/app/features/prepare-html-for-translate/utils/extractArticle.ts similarity index 100% rename from web/app/feature/translate/utils/extractArticle.ts rename to web/app/features/prepare-html-for-translate/utils/extractArticle.ts diff --git a/web/app/feature/translate/utils/extractNumberedElements.ts b/web/app/features/prepare-html-for-translate/utils/extractNumberedElements.ts similarity index 100% rename from web/app/feature/translate/utils/extractNumberedElements.ts rename to web/app/features/prepare-html-for-translate/utils/extractNumberedElements.ts diff --git a/web/app/feature/translate/utils/fetchWithRetry.ts b/web/app/features/prepare-html-for-translate/utils/fetchWithRetry.ts similarity index 100% rename from web/app/feature/translate/utils/fetchWithRetry.ts rename to web/app/features/prepare-html-for-translate/utils/fetchWithRetry.ts diff --git a/web/app/feature/translate/components/AIModelSelector.tsx b/web/app/features/translate/components/AIModelSelector.tsx similarity index 100% rename from web/app/feature/translate/components/AIModelSelector.tsx rename to web/app/features/translate/components/AIModelSelector.tsx diff --git a/web/app/routes/translate/constants.ts b/web/app/features/translate/constants.ts similarity index 100% rename from web/app/routes/translate/constants.ts rename to web/app/features/translate/constants.ts diff --git a/web/app/features/translate/functions/mutations.server.ts b/web/app/features/translate/functions/mutations.server.ts new file mode 100644 index 00000000..71ea4ea4 --- /dev/null +++ b/web/app/features/translate/functions/mutations.server.ts @@ -0,0 +1,184 @@ +import { createHash } from "node:crypto"; +import { prisma } from "~/utils/prisma"; + +export async function getOrCreateSourceTextIdAndPageVersionSourceText( + text: string, + number: number, + pageVersionId: number, +): Promise<{ id: number; number: number }> { + const textHash = Buffer.from( + createHash("sha256").update(text).digest("hex"), + "hex", + ); + + return prisma.$transaction(async (tx) => { + const sourceText = await tx.sourceText.upsert({ + where: { + textHash_number: { + textHash, + number, + }, + }, + update: {}, + create: { + text, + number, + textHash, + }, + }); + + await tx.pageVersionSourceText.upsert({ + where: { + pageVersionId_sourceTextId: { + pageVersionId, + sourceTextId: sourceText.id, + }, + }, + update: {}, + create: { + pageVersionId, + sourceTextId: sourceText.id, + }, + }); + + return { id: sourceText.id, number: sourceText.number }; + }); +} + +export async function getOrCreateAIUser(name: string): Promise { + const user = await prisma.user.upsert({ + where: { email: `${name}@ai.com` }, + update: {}, + create: { name, email: `${name}@ai.com`, isAI: true, image: "" }, + }); + + return user.id; +} + +export async function getOrCreatePageVersionTranslationInfo( + pageVersionId: number, + targetLanguage: string, + translationTitle: string, +) { + return await prisma.pageVersionTranslationInfo.upsert({ + where: { + pageVersionId_targetLanguage: { + pageVersionId, + targetLanguage, + }, + }, + update: {}, + create: { + pageVersionId, + targetLanguage, + translationTitle, + }, + }); +} + +export async function getOrCreatePageId(url: string): Promise { + const page = await prisma.page.upsert({ + where: { url }, + update: {}, + create: { url }, + }); + return page.id; +} + + +export async function getOrCreatePageVersionId( + url: string, + title: string, + content: string, + pageId: number, +): Promise { + const normalizedContent = content.trim().replace(/\s+/g, " "); + const contentHash = Buffer.from( + createHash("sha256").update(normalizedContent).digest("hex"), + "hex", + ); + const existingVersion = await prisma.pageVersion.findFirst({ + where: { + url, + contentHash, + }, + }); + + if (existingVersion) { + return existingVersion.id; + } + + const newVersion = await prisma.pageVersion.create({ + data: { + title, + url, + content, + contentHash, + page: { + connect: { + id: pageId, + }, + }, + }, + }); + + console.log(`New PageVersion created: ${newVersion.title}`); + return newVersion.id; +} + +export async function updateUserAITranslationInfo( + userId: number, + pageVersionId: number, + targetLanguage: string, + status: string, + progress: number, +) { + return await prisma.userAITranslationInfo.update({ + where: { + userId_pageVersionId_targetLanguage: { + userId, + pageVersionId, + targetLanguage, + }, + }, + data: { + aiTranslationStatus: status, + aiTranslationProgress: progress, + lastTranslatedAt: new Date(), + }, + }); +} + + +export async function getOrCreateUserAITranslationInfo( + userId: number, + pageVersionId: number, + targetLanguage: string, +) { + try { + const userAITranslationInfo = await prisma.userAITranslationInfo.upsert({ + where: { + userId_pageVersionId_targetLanguage: { + userId, + pageVersionId, + targetLanguage, + }, + }, + update: { + aiTranslationStatus: "pending", + aiTranslationProgress: 0, + }, + create: { + userId, + pageVersionId, + targetLanguage, + aiTranslationStatus: "pending", + aiTranslationProgress: 0, + }, + }); + return userAITranslationInfo; + } catch (error) { + console.error("Error in getOrCreateUserAITranslationInfo:", error); + throw error; + } +} diff --git a/web/app/feature/translate/jobs/translate-job.server.ts b/web/app/features/translate/jobs/translate-job.server.ts similarity index 60% rename from web/app/feature/translate/jobs/translate-job.server.ts rename to web/app/features/translate/jobs/translate-job.server.ts index 5f580365..79e08753 100644 --- a/web/app/feature/translate/jobs/translate-job.server.ts +++ b/web/app/features/translate/jobs/translate-job.server.ts @@ -1,8 +1,8 @@ -import { translate } from "../libs/translation"; -import { addNumbersToContent } from "../utils/addNumbersToContent"; -import { extractArticle } from "../utils/extractArticle"; -import { extractNumberedElements } from "../utils/extractNumberedElements"; -import { fetchWithRetry } from "../utils/fetchWithRetry"; +import { translate } from "../lib/translation"; +import { addNumbersToContent } from "../../prepare-html-for-translate/utils/addNumbersToContent"; +import { extractArticle } from "../../prepare-html-for-translate/utils/extractArticle"; +import { extractNumberedElements } from "../../prepare-html-for-translate/utils/extractNumberedElements"; +import { fetchWithRetry } from "../../prepare-html-for-translate/utils/fetchWithRetry"; interface TranslateJobParams { url: string; @@ -17,7 +17,6 @@ export const translateJob = async (params: TranslateJobParams) => { const numberedContent = addNumbersToContent(content); const extractedNumberedElements = extractNumberedElements( numberedContent, - title, ); const targetLanguage = params.targetLanguage; diff --git a/web/app/features/translate/lib/translation.ts b/web/app/features/translate/lib/translation.ts new file mode 100644 index 00000000..e00f76ea --- /dev/null +++ b/web/app/features/translate/lib/translation.ts @@ -0,0 +1,204 @@ +import { getOrCreatePageId } from "../functions/mutations.server"; +import { getOrCreatePageVersionId } from "../functions/mutations.server"; +import { getOrCreateUserAITranslationInfo } from "../functions/mutations.server"; +import { updateUserAITranslationInfo } from "../functions/mutations.server"; +import type { NumberedElement } from "../types"; +import { splitNumberedElements } from "../utils/splitNumberedElements.server"; +import { prisma } from "../../../utils/prisma"; +import { getOrCreateAIUser } from "../functions/mutations.server"; +import { getOrCreatePageVersionTranslationInfo } from "../functions/mutations.server"; +import { getOrCreateSourceTextIdAndPageVersionSourceText } from "../functions/mutations.server"; +import { getGeminiModelResponse } from "../services/gemini"; +import { extractTranslations } from "../utils/extractTranslations.server"; + +export async function translate( + geminiApiKey: string, + aiModel: string, + userId: number, + targetLanguage: string, + title: string, + numberedContent: string, + numberedElements: NumberedElement[], + url: string, +): Promise { + const pageId = await getOrCreatePageId(url || ""); + const pageVersionId = await getOrCreatePageVersionId( + url, + title, + numberedContent, + pageId, + ); + + const userAITranslationInfo = await getOrCreateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + ); + + await processTranslation( + geminiApiKey, + aiModel, + userId, + pageVersionId, + targetLanguage, + title, + numberedElements, + ); + + return userAITranslationInfo.aiTranslationStatus; +} + +export async function processTranslation( + geminiApiKey: string, + aiModel: string, + userId: number, + pageVersionId: number, + targetLanguage: string, + title: string, + numberedElements: NumberedElement[], +) { + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "in_progress", + 0, + ); + try { + const chunks = splitNumberedElements(numberedElements); + const totalChunks = chunks.length; + for (let i = 0; i < chunks.length; i++) { + console.log(`Processing chunk ${i + 1} of ${totalChunks}`); + + await translateChunk( + geminiApiKey, + aiModel, + chunks[i], + targetLanguage, + pageVersionId, + title, + ); + const progress = ((i + 1) / totalChunks) * 100; + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "in_progress", + progress, + ); + } + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "completed", + 100, + ); + } catch (error) { + console.error("Background translation job failed:", error); + await updateUserAITranslationInfo( + userId, + pageVersionId, + targetLanguage, + "failed", + 0, + ); + } +} + +export async function translateChunk( + geminiApiKey: string, + aiModel: string, + numberedElements: NumberedElement[], + targetLanguage: string, + pageVersionId: number, + title: string, +) { + const sourceTexts = await getSourceTexts(numberedElements, pageVersionId); + const translatedText = await getTranslatedText( + geminiApiKey, + aiModel, + numberedElements, + targetLanguage, + title, + ); + + const extractedTranslations = extractTranslations(translatedText); + await getOrCreatePageVersionTranslationInfo( + pageVersionId, + targetLanguage, + extractedTranslations[0].text, + ); + + await saveTranslations( + extractedTranslations, + sourceTexts, + targetLanguage, + aiModel, + ); +} +async function getSourceTexts( + numberedElements: NumberedElement[], + pageVersionId: number, +) { + return Promise.all( + numberedElements.map((element) => + getOrCreateSourceTextIdAndPageVersionSourceText( + element.text, + element.number, + pageVersionId, + ), + ), + ); +} +async function getTranslatedText( + geminiApiKey: string, + aiModel: string, + numberedElements: NumberedElement[], + targetLanguage: string, + title: string, +) { + const source_text = numberedElements + .map((el) => JSON.stringify(el)) + .join("\n"); + return getGeminiModelResponse( + geminiApiKey, + aiModel, + title, + source_text, + targetLanguage, + ); +} + +async function saveTranslations( + extractedTranslations: NumberedElement[], + sourceTexts: { id: number; number: number }[], + targetLanguage: string, + aiModel: string, +) { + const systemUserId = await getOrCreateAIUser(aiModel); + + const translationData = extractedTranslations + .map((translation) => { + const sourceTextId = sourceTexts.find( + (sourceText) => sourceText.number === translation.number, + )?.id; + if (!sourceTextId) { + console.error( + `Source text ID not found for translation number ${translation.number}`, + ); + return null; + } + return { + targetLanguage, + text: translation.text, + sourceTextId, + userId: systemUserId, + }; + }) + .filter((item): item is NonNullable => item !== null); + + if (translationData.length > 0) { + await prisma.translateText.createMany({ data: translationData }); + } +} diff --git a/web/app/feature/translate/utils/anthropic.ts b/web/app/features/translate/services/anthropic.ts similarity index 87% rename from web/app/feature/translate/utils/anthropic.ts rename to web/app/features/translate/services/anthropic.ts index 722c6e6f..86d63a09 100644 --- a/web/app/feature/translate/utils/anthropic.ts +++ b/web/app/features/translate/services/anthropic.ts @@ -1,5 +1,5 @@ import Anthropic from "@anthropic-ai/sdk"; -import { generateSystemMessage } from "../utils/generateAnthropicSystemMessage"; +import { generateSystemMessage } from "./generateAnthropicSystemMessage"; export async function callAnthropic( apiKey: string | undefined, diff --git a/web/app/feature/translate/utils/gemini.ts b/web/app/features/translate/services/gemini.ts similarity index 100% rename from web/app/feature/translate/utils/gemini.ts rename to web/app/features/translate/services/gemini.ts diff --git a/web/app/feature/translate/utils/generateGeminiMessage.ts b/web/app/features/translate/services/generateGeminiMessage.ts similarity index 100% rename from web/app/feature/translate/utils/generateGeminiMessage.ts rename to web/app/features/translate/services/generateGeminiMessage.ts diff --git a/web/app/feature/translate/utils/openai.ts b/web/app/features/translate/services/openai.ts similarity index 90% rename from web/app/feature/translate/utils/openai.ts rename to web/app/features/translate/services/openai.ts index 45104939..ae255614 100644 --- a/web/app/feature/translate/utils/openai.ts +++ b/web/app/features/translate/services/openai.ts @@ -1,6 +1,6 @@ import { OpenAI } from "openai/index.mjs"; import type { TranslationRequestBody } from "../schemas/translation-request.js"; -import { generateSystemMessage } from "../utils/generateAnthropicSystemMessage.js"; +import { generateSystemMessage } from "./generateAnthropicSystemMessage.js"; export async function createTranslation( apiKey: string | undefined, diff --git a/web/app/feature/translate/utils/vertexai.ts b/web/app/features/translate/services/vertexai.ts similarity index 100% rename from web/app/feature/translate/utils/vertexai.ts rename to web/app/features/translate/services/vertexai.ts diff --git a/web/app/feature/translate/translate-user-queue.ts b/web/app/features/translate/translate-user-queue.ts similarity index 97% rename from web/app/feature/translate/translate-user-queue.ts rename to web/app/features/translate/translate-user-queue.ts index 00439ee4..c9b4cd91 100644 --- a/web/app/feature/translate/translate-user-queue.ts +++ b/web/app/features/translate/translate-user-queue.ts @@ -9,7 +9,7 @@ type TranslateJobData = { aiModel: string; }; -const QUEUE_VERSION = 1; +const QUEUE_VERSION = 140; export const getTranslateUserQueue = (userId: number) => { return Queue(`translation-user-${userId}`, QUEUE_VERSION, { diff --git a/web/app/features/translate/types.ts b/web/app/features/translate/types.ts new file mode 100644 index 00000000..a4ef0140 --- /dev/null +++ b/web/app/features/translate/types.ts @@ -0,0 +1,4 @@ +export type NumberedElement = { + number: number; + text: string; +}; diff --git a/web/app/features/translate/utils/extractTranslations.server.ts b/web/app/features/translate/utils/extractTranslations.server.ts new file mode 100644 index 00000000..29127280 --- /dev/null +++ b/web/app/features/translate/utils/extractTranslations.server.ts @@ -0,0 +1,30 @@ + +export function extractTranslations( + text: string, +): { number: number; text: string }[] { + try { + const parsed = JSON.parse(text); + if (Array.isArray(parsed)) { + return parsed; + } + } catch (error) { + console.warn("Failed to parse JSON, falling back to regex parsing", error); + } + + const translations: { number: number; text: string }[] = []; + const regex = + /{\s*"number"\s*:\s*(\d+)\s*,\s*"text"\s*:\s*"((?:\\.|[^"\\])*)"\s*}/g; + let match: RegExpExecArray | null; + + while (true) { + match = regex.exec(text); + if (match === null) break; + + translations.push({ + number: Number.parseInt(match[1], 10), + text: match[2], + }); + } + + return translations; +} \ No newline at end of file diff --git a/web/app/features/translate/utils/splitNumberedElements.server.ts b/web/app/features/translate/utils/splitNumberedElements.server.ts new file mode 100644 index 00000000..8386e7d4 --- /dev/null +++ b/web/app/features/translate/utils/splitNumberedElements.server.ts @@ -0,0 +1,30 @@ +import { MAX_CHUNK_SIZE } from "../constants"; +import type { NumberedElement } from "../types"; + + +export function splitNumberedElements( + elements: NumberedElement[], +): NumberedElement[][] { + const chunks: NumberedElement[][] = []; + let currentChunk: NumberedElement[] = []; + let currentSize = 0; + + for (const element of elements) { + if ( + currentSize + element.text.length > MAX_CHUNK_SIZE && + currentChunk.length > 0 + ) { + chunks.push(currentChunk); + currentChunk = []; + currentSize = 0; + } + currentChunk.push(element); + currentSize += element.text.length; + } + + if (currentChunk.length > 0) { + chunks.push(currentChunk); + } + return chunks; +} + diff --git a/web/app/libs/db/user.server.ts b/web/app/libs/db/user.server.ts deleted file mode 100644 index 08e36d42..00000000 --- a/web/app/libs/db/user.server.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { prisma } from "~/utils/prisma"; - -export const getDbUser = async (userId: number) => { - return await prisma.user.findUnique({ where: { id: userId } }); -}; - -export async function getOrCreateAIUser(name: string): Promise { - const user = await prisma.user.upsert({ - where: { email: `${name}@ai.com` }, - update: {}, - create: { name, email: `${name}@ai.com`, isAI: true, image: "" }, - }); - - return user.id; -} diff --git a/web/app/libs/pageService.ts b/web/app/libs/pageService.ts deleted file mode 100644 index a9997afe..00000000 --- a/web/app/libs/pageService.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { prisma } from "../utils/prisma"; - -export async function getOrCreatePageId(url: string): Promise { - const page = await prisma.page.upsert({ - where: { url }, - update: {}, - create: { url }, - }); - return page.id; -} diff --git a/web/app/libs/pageVersion.ts b/web/app/libs/pageVersion.ts deleted file mode 100644 index 49d2fd56..00000000 --- a/web/app/libs/pageVersion.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { createHash } from "node:crypto"; -import { prisma } from "../utils/prisma"; - -export async function getOrCreatePageVersionId( - url: string, - title: string, - content: string, - pageId: number, -): Promise { - const normalizedContent = content.trim().replace(/\s+/g, " "); - const contentHash = Buffer.from( - createHash("sha256").update(normalizedContent).digest("hex"), - "hex", - ); - const existingVersion = await prisma.pageVersion.findFirst({ - where: { - url, - contentHash, - }, - }); - - if (existingVersion) { - return existingVersion.id; - } - - const newVersion = await prisma.pageVersion.create({ - data: { - title, - url, - content, - contentHash, - page: { - connect: { - id: pageId, - }, - }, - }, - }); - - console.log(`New PageVersion created: ${newVersion.title}`); - return newVersion.id; -} diff --git a/web/app/libs/pageVersionTranslationInfo.ts b/web/app/libs/pageVersionTranslationInfo.ts deleted file mode 100644 index d88f93a6..00000000 --- a/web/app/libs/pageVersionTranslationInfo.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { prisma } from "~/utils/prisma"; - -export async function getOrCreatePageVersionTranslationInfo( - pageVersionId: number, - targetLanguage: string, - translationTitle: string, -) { - return await prisma.pageVersionTranslationInfo.upsert({ - where: { - pageVersionId_targetLanguage: { - pageVersionId, - targetLanguage, - }, - }, - update: {}, // 既存のレコードがある場合は更新しない - create: { - pageVersionId, - targetLanguage, - translationTitle, - }, - }); -} diff --git a/web/app/libs/userAITranslationInfo.tsx b/web/app/libs/userAITranslationInfo.tsx deleted file mode 100644 index eb4e6a87..00000000 --- a/web/app/libs/userAITranslationInfo.tsx +++ /dev/null @@ -1,57 +0,0 @@ -import { prisma } from "~/utils/prisma"; - -export async function getOrCreateUserAITranslationInfo( - userId: number, - pageVersionId: number, - targetLanguage: string, -) { - try { - const userAITranslationInfo = await prisma.userAITranslationInfo.upsert({ - where: { - userId_pageVersionId_targetLanguage: { - userId, - pageVersionId, - targetLanguage, - }, - }, - update: { - aiTranslationStatus: "pending", - aiTranslationProgress: 0, - }, - create: { - userId, - pageVersionId, - targetLanguage, - aiTranslationStatus: "pending", - aiTranslationProgress: 0, - }, - }); - return userAITranslationInfo; - } catch (error) { - console.error("Error in getOrCreateUserAITranslationInfo:", error); - throw error; - } -} - -export async function updateUserAITranslationInfo( - userId: number, - pageVersionId: number, - targetLanguage: string, - status: string, - progress: number, -) { - return await prisma.userAITranslationInfo.update({ - where: { - userId_pageVersionId_targetLanguage: { - userId, - pageVersionId, - targetLanguage, - }, - }, - data: { - aiTranslationStatus: status, - aiTranslationProgress: progress, - lastTranslatedAt: new Date(), // 明示的に更新 - }, - }); -} diff --git a/web/app/libs/userReadHistory.ts b/web/app/libs/userReadHistory.ts deleted file mode 100644 index db620277..00000000 --- a/web/app/libs/userReadHistory.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { prisma } from "../utils/prisma"; - -export async function updateUserReadHistory( - userId: number, - pageVersionId: number, - lastReadDataNumber: number, -) { - await prisma.userReadHistory.upsert({ - where: { - userId_pageVersionId: { - userId: userId, - pageVersionId: pageVersionId, - }, - }, - update: { - lastReadDataNumber: lastReadDataNumber, - readAt: new Date(), - }, - create: { - userId: userId, - pageVersionId: pageVersionId, - lastReadDataNumber: lastReadDataNumber, - }, - }); -} - -export async function getLastReadDataNumber( - userId: number, - pageVersionId: number, -) { - const readHistory = await prisma.userReadHistory.findUnique({ - where: { - userId_pageVersionId: { - userId: userId, - pageVersionId: pageVersionId, - }, - }, - select: { - lastReadDataNumber: true, - }, - }); - - return readHistory?.lastReadDataNumber ?? 0; -} diff --git a/web/app/routes/_index/route.tsx b/web/app/routes/_index/route.tsx index 17831c3e..54f028fb 100644 --- a/web/app/routes/_index/route.tsx +++ b/web/app/routes/_index/route.tsx @@ -29,7 +29,7 @@ export async function loader({ request }: LoaderFunctionArgs) { export async function action({ request }: ActionFunctionArgs) { return authenticator.authenticate("google", request, { - successRedirect: "/translate", + successRedirect: "/translator", failureRedirect: "/", }); } diff --git a/web/app/routes/api.auth.callback.google.ts b/web/app/routes/api.auth.callback.google.ts index e151e522..c7ffed99 100644 --- a/web/app/routes/api.auth.callback.google.ts +++ b/web/app/routes/api.auth.callback.google.ts @@ -4,7 +4,7 @@ import { authenticator } from "../utils/auth.server"; export const loader = ({ request }: LoaderFunctionArgs) => { try { return authenticator.authenticate("google", request, { - successRedirect: "/translate", + successRedirect: "/translator", failureRedirect: "/", }); } catch (error) { diff --git a/web/app/routes/reader.$/components/Translation.tsx b/web/app/routes/reader.$/components/Translation.tsx index fb3fe98e..9ec2a91d 100644 --- a/web/app/routes/reader.$/components/Translation.tsx +++ b/web/app/routes/reader.$/components/Translation.tsx @@ -1,7 +1,7 @@ import { FilePenLine, X } from "lucide-react"; import { useMemo, useState } from "react"; -import { getBestTranslation } from "../libs/get-best-translation.client"; -import { sanitizeAndParseText } from "../libs/sanitize-and-parse-text.client"; +import { getBestTranslation } from "../lib/get-best-translation.client"; +import { sanitizeAndParseText } from "../lib/sanitize-and-parse-text.client"; import type { TranslationWithVote } from "../types"; import { AddAndVoteTranslations } from "./AddAndVoteTranslations"; interface TranslationProps { diff --git a/web/app/routes/reader.$/components/TranslationItem.tsx b/web/app/routes/reader.$/components/TranslationItem.tsx index f4f0e74e..31969cc7 100644 --- a/web/app/routes/reader.$/components/TranslationItem.tsx +++ b/web/app/routes/reader.$/components/TranslationItem.tsx @@ -1,4 +1,4 @@ -import { sanitizeAndParseText } from "../libs/sanitize-and-parse-text.client"; +import { sanitizeAndParseText } from "../lib/sanitize-and-parse-text.client"; import type { TranslationWithVote } from "../types"; import { VoteButtons } from "./VoteButtons"; diff --git a/web/app/routes/reader.$/functions/mutations.server.ts b/web/app/routes/reader.$/functions/mutations.server.ts index 3c508f00..b4c0ff79 100644 --- a/web/app/routes/reader.$/functions/mutations.server.ts +++ b/web/app/routes/reader.$/functions/mutations.server.ts @@ -67,3 +67,28 @@ export async function handleAddTranslationAction( return json({ success: true }); } + + +export async function updateUserReadHistory( + userId: number, + pageVersionId: number, + lastReadDataNumber: number, +) { + await prisma.userReadHistory.upsert({ + where: { + userId_pageVersionId: { + userId: userId, + pageVersionId: pageVersionId, + }, + }, + update: { + lastReadDataNumber: lastReadDataNumber, + readAt: new Date(), + }, + create: { + userId: userId, + pageVersionId: pageVersionId, + lastReadDataNumber: lastReadDataNumber, + }, + }); +} diff --git a/web/app/routes/reader.$/functions/queries.server.ts b/web/app/routes/reader.$/functions/queries.server.ts index 76ccd8bc..119caa55 100644 --- a/web/app/routes/reader.$/functions/queries.server.ts +++ b/web/app/routes/reader.$/functions/queries.server.ts @@ -77,3 +77,24 @@ export async function fetchLatestPageVersionWithTranslations( userId, }; } + + + +export async function getLastReadDataNumber( + userId: number, + pageVersionId: number, +) { + const readHistory = await prisma.userReadHistory.findUnique({ + where: { + userId_pageVersionId: { + userId: userId, + pageVersionId: pageVersionId, + }, + }, + select: { + lastReadDataNumber: true, + }, + }); + + return readHistory?.lastReadDataNumber ?? 0; +} diff --git a/web/app/routes/reader.$/libs/get-best-translation.client.ts b/web/app/routes/reader.$/lib/get-best-translation.client.ts similarity index 100% rename from web/app/routes/reader.$/libs/get-best-translation.client.ts rename to web/app/routes/reader.$/lib/get-best-translation.client.ts diff --git a/web/app/routes/reader.$/libs/sanitize-and-parse-text.client.ts b/web/app/routes/reader.$/lib/sanitize-and-parse-text.client.ts similarity index 100% rename from web/app/routes/reader.$/libs/sanitize-and-parse-text.client.ts rename to web/app/routes/reader.$/lib/sanitize-and-parse-text.client.ts diff --git a/web/app/routes/reader.$/route.tsx b/web/app/routes/reader.$/route.tsx index e6f061b8..eaed02bb 100644 --- a/web/app/routes/reader.$/route.tsx +++ b/web/app/routes/reader.$/route.tsx @@ -9,9 +9,7 @@ import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { Header } from "~/components/Header"; import { LoadingSpinner } from "~/components/LoadingSpinner"; import { Button } from "~/components/ui/button"; -import { AIModelSelector } from "~/feature/translate/components/AIModelSelector"; -import { getTranslateUserQueue } from "~/feature/translate/translate-user-queue"; -import { getDbUser } from "~/libs/db/user.server"; +import { AIModelSelector } from "~/features/translate/components/AIModelSelector"; import { normalizeAndSanitizeUrl } from "~/utils/normalize-and-sanitize-url.server"; import { getTargetLanguage } from "~/utils/target-language.server"; import { authenticator } from "../../utils/auth.server"; @@ -82,30 +80,6 @@ export const action = async ({ request }: ActionFunctionArgs) => { targetLanguage, ); return { intent, lastResult: submission.reply({ resetForm: true }) }; - case "retranslate": { - const dbUser = await getDbUser(safeUser.id); - if (!dbUser?.geminiApiKey) { - return { - lastResult: submission.reply({ - formErrors: ["Gemini API key is not set"], - }), - url: null, - }; - } - const normalizedUrl = normalizeAndSanitizeUrl(submission.value.url); - // Start the translation job in background - const queue = getTranslateUserQueue(safeUser.id); - const job = await queue.add(`translate-${safeUser.id}`, { - url: normalizedUrl, - targetLanguage, - apiKey: dbUser.geminiApiKey, - userId: safeUser.id, - aiModel: submission.value.aiModel, - }); - console.log(job.toJSON()); - - return { intent, lastResult: submission.reply({ resetForm: true }) }; - } default: throw new Error("Invalid Intent"); } @@ -128,7 +102,7 @@ export default function ReaderView() {
-
+
diff --git a/web/app/routes/resources+/gemini-api-key-form.tsx b/web/app/routes/resources+/gemini-api-key-form.tsx index 9cbffb4d..09bdb09a 100644 --- a/web/app/routes/resources+/gemini-api-key-form.tsx +++ b/web/app/routes/resources+/gemini-api-key-form.tsx @@ -13,7 +13,7 @@ import { Alert, AlertDescription } from "~/components/ui/alert"; import { Button } from "~/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { Input } from "~/components/ui/input"; -import { validateGeminiApiKey } from "~/feature/translate/utils/gemini"; +import { validateGeminiApiKey } from "~/features/translate/services/gemini"; import { authenticator } from "~/utils/auth.server"; import { updateGeminiApiKey } from "./functions/mutations.server"; diff --git a/web/app/routes/translate/components/URLTranslationForm.tsx b/web/app/routes/translator/components/URLTranslationForm.tsx similarity index 96% rename from web/app/routes/translate/components/URLTranslationForm.tsx rename to web/app/routes/translator/components/URLTranslationForm.tsx index 5a091171..3248e6dd 100644 --- a/web/app/routes/translate/components/URLTranslationForm.tsx +++ b/web/app/routes/translator/components/URLTranslationForm.tsx @@ -8,7 +8,7 @@ import { LoadingSpinner } from "~/components/LoadingSpinner"; import { Alert, AlertDescription, AlertTitle } from "~/components/ui/alert"; import { Button } from "~/components/ui/button"; import { Input } from "~/components/ui/input"; -import { AIModelSelector } from "~/feature/translate/components/AIModelSelector"; +import { AIModelSelector } from "~/features/translate/components/AIModelSelector"; import type { action } from "../route"; import { urlTranslationSchema } from "../types"; diff --git a/web/app/routes/translate/components/UserAITranslationStatus.tsx b/web/app/routes/translator/components/UserAITranslationStatus.tsx similarity index 97% rename from web/app/routes/translate/components/UserAITranslationStatus.tsx rename to web/app/routes/translator/components/UserAITranslationStatus.tsx index f0feaf85..309aae30 100644 --- a/web/app/routes/translate/components/UserAITranslationStatus.tsx +++ b/web/app/routes/translator/components/UserAITranslationStatus.tsx @@ -11,7 +11,7 @@ import { Button } from "~/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "~/components/ui/card"; import { Progress } from "~/components/ui/progress"; import { ScrollArea } from "~/components/ui/scroll-area"; -import { AIModelSelector } from "~/feature/translate/components/AIModelSelector"; +import { AIModelSelector } from "~/features/translate/components/AIModelSelector"; import { cn } from "~/utils/cn"; import type { UserAITranslationInfoItem } from "../types"; import { urlTranslationSchema } from "../types"; @@ -89,7 +89,7 @@ export function UserAITranslationStatus({ className={cn( "mt-2", item.aiTranslationStatus === "in_progress" && - "bg-blue-400 animate-pulse", + "bg-blue-400 animate-pulse", )} />

diff --git a/web/app/routes/translate/functions/queries.server.ts b/web/app/routes/translator/functions/queries.server.ts similarity index 100% rename from web/app/routes/translate/functions/queries.server.ts rename to web/app/routes/translator/functions/queries.server.ts diff --git a/web/app/routes/translate/route.tsx b/web/app/routes/translator/route.tsx similarity index 96% rename from web/app/routes/translate/route.tsx rename to web/app/routes/translator/route.tsx index 9123bac0..cba81b55 100644 --- a/web/app/routes/translate/route.tsx +++ b/web/app/routes/translator/route.tsx @@ -4,8 +4,8 @@ import { useRevalidator } from "@remix-run/react"; import { useEffect } from "react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { Header } from "~/components/Header"; -import { getTranslateUserQueue } from "~/feature/translate/translate-user-queue"; -import { getDbUser } from "~/libs/db/user.server"; +import { getTranslateUserQueue } from "~/features/translate/translate-user-queue"; +import { getDbUser } from "./functions/queries.server"; import { authenticator } from "~/utils/auth.server"; import { normalizeAndSanitizeUrl } from "~/utils/normalize-and-sanitize-url.server"; import { getTargetLanguage } from "~/utils/target-language.server"; diff --git a/web/app/routes/translate/types.ts b/web/app/routes/translator/types.ts similarity index 94% rename from web/app/routes/translate/types.ts rename to web/app/routes/translator/types.ts index d86dd5d8..64e297a4 100644 --- a/web/app/routes/translate/types.ts +++ b/web/app/routes/translator/types.ts @@ -41,7 +41,3 @@ export type PageVersionTranslationInfoItem = z.infer< typeof PageVersionTranslationInfoSchema >; -export type NumberedElement = { - number: number; - text: string; -};