From ee0df7caf2f2fe88945e1e6167567a446a9d49f7 Mon Sep 17 00:00:00 2001 From: tomolld Date: Sat, 27 Jul 2024 21:57:21 +0900 Subject: [PATCH 1/4] chore: Update npm dependencies and add normalize-url package --- .../components/ContentWithTranslations.tsx | 5 +--- .../components/Translation.tsx | 10 +++---- .../functions/queries.server.ts | 6 +++- web/app/routes/reader.$encodedUrl/route.tsx | 8 +++-- web/app/routes/translate/route.tsx | 6 ++-- .../normalize-and-sanitize-url.server.ts | 29 +++++++++++++++++++ web/package.json | 1 + 7 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 web/app/utils/normalize-and-sanitize-url.server.ts diff --git a/web/app/routes/reader.$encodedUrl/components/ContentWithTranslations.tsx b/web/app/routes/reader.$encodedUrl/components/ContentWithTranslations.tsx index 747fc1ca..f4f2cd7d 100644 --- a/web/app/routes/reader.$encodedUrl/components/ContentWithTranslations.tsx +++ b/web/app/routes/reader.$encodedUrl/components/ContentWithTranslations.tsx @@ -7,14 +7,12 @@ import { Translation } from "./Translation"; interface ContentWithTranslationsProps { content: string; sourceTextInfoWithTranslations: SourceTextInfoWithTranslations[]; - targetLanguage: string; userId: number | null; } export const ContentWithTranslations = memo(function ContentWithTranslations({ content, sourceTextInfoWithTranslations, - targetLanguage, userId, }: ContentWithTranslationsProps) { const parsedContent = useMemo(() => { @@ -53,7 +51,6 @@ export const ContentWithTranslations = memo(function ContentWithTranslations({ @@ -63,7 +60,7 @@ export const ContentWithTranslations = memo(function ContentWithTranslations({ return domNode; }, }); - }, [content, sourceTextInfoWithTranslations, targetLanguage, userId]); + }, [content, sourceTextInfoWithTranslations, userId]); if (typeof window === "undefined") { return
Loading...
; diff --git a/web/app/routes/reader.$encodedUrl/components/Translation.tsx b/web/app/routes/reader.$encodedUrl/components/Translation.tsx index 684a980f..a79853c0 100644 --- a/web/app/routes/reader.$encodedUrl/components/Translation.tsx +++ b/web/app/routes/reader.$encodedUrl/components/Translation.tsx @@ -6,7 +6,6 @@ import { AddAndVoteTranslations } from "./AddAndVoteTranslations"; interface TranslationProps { translationsWithVotes: TranslationWithVote[]; - targetLanguage: string; userId: number | null; sourceTextId: number; } @@ -31,11 +30,10 @@ function getBestTranslation( export function Translation({ translationsWithVotes, - targetLanguage, userId, sourceTextId, }: TranslationProps) { - const [isLoaded, setIsLoaded] = useState(false); + const [isHovered, setIsHovered] = useState(false); const bestTranslationWithVote = useMemo( () => getBestTranslation(translationsWithVotes), @@ -58,13 +56,15 @@ export function Translation({ return (
!isLoaded && setIsLoaded(true)} + onMouseEnter={() => + !isHovered && setTimeout(() => setIsHovered(true), 500) + } >
{sanitizedAndParsedText}
- {isLoaded && ( + {isHovered && ( { const pageVersion = await prisma.pageVersion.findFirst({ - where: { url }, + where: { + url: { + contains: url, + }, + }, orderBy: { createdAt: "desc" }, select: { title: true, diff --git a/web/app/routes/reader.$encodedUrl/route.tsx b/web/app/routes/reader.$encodedUrl/route.tsx index 9faa6980..8c7d3dfe 100644 --- a/web/app/routes/reader.$encodedUrl/route.tsx +++ b/web/app/routes/reader.$encodedUrl/route.tsx @@ -3,6 +3,7 @@ import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; import { useParams } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { Header } from "~/components/Header"; +import { prepareUrlForSearchFromRawInput } from "~/utils/normalize-and-sanitize-url.server"; import { getTargetLanguage } from "~/utils/target-language.server"; import { authenticator } from "../../utils/auth.server"; import { ContentWithTranslations } from "./components/ContentWithTranslations"; @@ -15,15 +16,17 @@ import { actionSchema } from "./types"; export const loader = async ({ params, request }: LoaderFunctionArgs) => { const targetLanguage = await getTargetLanguage(request); - const { encodedUrl } = params; + if (!encodedUrl) { throw new Response("Missing URL parameter", { status: 400 }); } + const safeUser = await authenticator.isAuthenticated(request); const safeUserId = safeUser?.id; + const searchUrl = prepareUrlForSearchFromRawInput(encodedUrl); const pageData = await fetchLatestPageVersionWithTranslations( - decodeURIComponent(encodedUrl), + searchUrl, safeUserId ?? 0, targetLanguage, ); @@ -107,7 +110,6 @@ export default function ReaderView() { sourceTextInfoWithTranslations={ pageData.sourceTextInfoWithTranslations } - targetLanguage={targetLanguage} userId={safeUser?.id ?? null} /> diff --git a/web/app/routes/translate/route.tsx b/web/app/routes/translate/route.tsx index c8612860..79a7a8f4 100644 --- a/web/app/routes/translate/route.tsx +++ b/web/app/routes/translate/route.tsx @@ -7,6 +7,7 @@ import { Header } from "~/components/Header"; import { getTranslateUserQueue } from "~/feature/translate/translate-user-queue"; import { validateGeminiApiKey } from "~/feature/translate/utils/gemini"; import { authenticator } from "~/utils/auth.server"; +import { normalizeAndSanitizeUrl } from "~/utils/normalize-and-sanitize-url.server"; import { getTargetLanguage } from "~/utils/target-language.server"; import { GeminiApiKeyForm } from "./components/GeminiApiKeyForm"; import { URLTranslationForm } from "./components/URLTranslationForm"; @@ -75,10 +76,11 @@ export async function action({ request }: ActionFunctionArgs) { }; } const targetLanguage = await getTargetLanguage(request); + const normalizedUrl = normalizeAndSanitizeUrl(submission.value.url); // Start the translation job in background const queue = getTranslateUserQueue(safeUser.id); const job = await queue.add(`translate-${safeUser.id}`, { - url: submission.value.url, + url: normalizedUrl, targetLanguage, apiKey: dbUser.geminiApiKey, userId: safeUser.id, @@ -88,7 +90,7 @@ export async function action({ request }: ActionFunctionArgs) { return { intent, lastResult: submission.reply({ resetForm: true }), - url: submission.value.url, + url: normalizedUrl, }; } default: { diff --git a/web/app/utils/normalize-and-sanitize-url.server.ts b/web/app/utils/normalize-and-sanitize-url.server.ts new file mode 100644 index 00000000..5730251e --- /dev/null +++ b/web/app/utils/normalize-and-sanitize-url.server.ts @@ -0,0 +1,29 @@ +import normalizeUrl from "normalize-url"; + +export function normalizeAndSanitizeUrl(inputUrl: string): string { + let decodedUrl: string; + try { + decodedUrl = decodeURIComponent(inputUrl); + } catch { + decodedUrl = inputUrl; + } + + return normalizeUrl(decodedUrl, { + stripHash: true, + removeQueryParameters: true, + }); +} + +export function prepareUrlForSearch(inputUrl: string): string { + let url = inputUrl; + url = url.replace(/^https?:\/\//, ""); + url = url.replace(/^www\./, ""); + url = url.split(/[?#]/)[0]; + url = url.replace(/\/$/, ""); + return url.toLowerCase(); +} + +export function prepareUrlForSearchFromRawInput(inputUrl: string): string { + const normalizedUrl = normalizeAndSanitizeUrl(inputUrl); + return prepareUrlForSearch(normalizedUrl); +} diff --git a/web/package.json b/web/package.json index 6aee1482..2b7e8035 100644 --- a/web/package.json +++ b/web/package.json @@ -47,6 +47,7 @@ "isomorphic-dompurify": "^2.13.0", "lucide-react": "^0.408.0", "next-themes": "^0.3.0", + "normalize-url": "^8.0.1", "react": "^19.0.0-beta-26f2496093-20240514", "react-dom": "^19.0.0-beta-26f2496093-20240514", "react-icons": "^5.2.1", From 79ea68389ff4809711368312c0df369ad1f2ad32 Mon Sep 17 00:00:00 2001 From: tomolld Date: Sat, 27 Jul 2024 23:20:01 +0900 Subject: [PATCH 2/4] f --- .../functions/queries.server.ts | 6 +- web/app/routes/reader.$encodedUrl/route.tsx | 6 +- .../normalize-and-sanitize-url.server.ts | 14 ---- web/scripts/normalize-page-urls.ts | 70 +++++++++++++++++++ 4 files changed, 74 insertions(+), 22 deletions(-) create mode 100644 web/scripts/normalize-page-urls.ts diff --git a/web/app/routes/reader.$encodedUrl/functions/queries.server.ts b/web/app/routes/reader.$encodedUrl/functions/queries.server.ts index d1219435..c013123a 100644 --- a/web/app/routes/reader.$encodedUrl/functions/queries.server.ts +++ b/web/app/routes/reader.$encodedUrl/functions/queries.server.ts @@ -10,11 +10,7 @@ export async function fetchLatestPageVersionWithTranslations( targetLanguage: string, ): Promise { const pageVersion = await prisma.pageVersion.findFirst({ - where: { - url: { - contains: url, - }, - }, + where: { url }, orderBy: { createdAt: "desc" }, select: { title: true, diff --git a/web/app/routes/reader.$encodedUrl/route.tsx b/web/app/routes/reader.$encodedUrl/route.tsx index 8c7d3dfe..8113cf5a 100644 --- a/web/app/routes/reader.$encodedUrl/route.tsx +++ b/web/app/routes/reader.$encodedUrl/route.tsx @@ -3,7 +3,7 @@ import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/node"; import { useParams } from "@remix-run/react"; import { typedjson, useTypedLoaderData } from "remix-typedjson"; import { Header } from "~/components/Header"; -import { prepareUrlForSearchFromRawInput } from "~/utils/normalize-and-sanitize-url.server"; +import { normalizeAndSanitizeUrl } from "~/utils/normalize-and-sanitize-url.server"; import { getTargetLanguage } from "~/utils/target-language.server"; import { authenticator } from "../../utils/auth.server"; import { ContentWithTranslations } from "./components/ContentWithTranslations"; @@ -24,9 +24,9 @@ export const loader = async ({ params, request }: LoaderFunctionArgs) => { const safeUser = await authenticator.isAuthenticated(request); const safeUserId = safeUser?.id; - const searchUrl = prepareUrlForSearchFromRawInput(encodedUrl); + const normalizedUrl = normalizeAndSanitizeUrl(encodedUrl); const pageData = await fetchLatestPageVersionWithTranslations( - searchUrl, + normalizedUrl, safeUserId ?? 0, targetLanguage, ); diff --git a/web/app/utils/normalize-and-sanitize-url.server.ts b/web/app/utils/normalize-and-sanitize-url.server.ts index 5730251e..16f9804e 100644 --- a/web/app/utils/normalize-and-sanitize-url.server.ts +++ b/web/app/utils/normalize-and-sanitize-url.server.ts @@ -13,17 +13,3 @@ export function normalizeAndSanitizeUrl(inputUrl: string): string { removeQueryParameters: true, }); } - -export function prepareUrlForSearch(inputUrl: string): string { - let url = inputUrl; - url = url.replace(/^https?:\/\//, ""); - url = url.replace(/^www\./, ""); - url = url.split(/[?#]/)[0]; - url = url.replace(/\/$/, ""); - return url.toLowerCase(); -} - -export function prepareUrlForSearchFromRawInput(inputUrl: string): string { - const normalizedUrl = normalizeAndSanitizeUrl(inputUrl); - return prepareUrlForSearch(normalizedUrl); -} diff --git a/web/scripts/normalize-page-urls.ts b/web/scripts/normalize-page-urls.ts new file mode 100644 index 00000000..47a69f16 --- /dev/null +++ b/web/scripts/normalize-page-urls.ts @@ -0,0 +1,70 @@ +import { PrismaClient } from "@prisma/client"; +import { normalizeAndSanitizeUrl } from "../app/utils/normalize-and-sanitize-url.server"; + +const prisma = new PrismaClient(); + +async function normalizeUrls() { + const batchSize = 100; + let processedPageCount = 0; + let processedPageVersionCount = 0; + + try { + // Normalize Page URLs + while (true) { + const pages = await prisma.page.findMany({ + take: batchSize, + skip: processedPageCount, + select: { id: true, url: true }, + }); + + if (pages.length === 0) break; + + for (const page of pages) { + const normalizedUrl = normalizeAndSanitizeUrl(page.url); + if (normalizedUrl !== page.url) { + await prisma.page.update({ + where: { id: page.id }, + data: { url: normalizedUrl }, + }); + console.log(`Updated Page URL: ${page.url} -> ${normalizedUrl}`); + } + } + + processedPageCount += pages.length; + console.log(`Processed ${processedPageCount} pages`); + } + + // Normalize PageVersion URLs + while (true) { + const pageVersions = await prisma.pageVersion.findMany({ + take: batchSize, + skip: processedPageVersionCount, + select: { id: true, url: true }, + }); + + if (pageVersions.length === 0) break; + + for (const pageVersion of pageVersions) { + const normalizedUrl = normalizeAndSanitizeUrl(pageVersion.url); + if (normalizedUrl !== pageVersion.url) { + await prisma.pageVersion.update({ + where: { id: pageVersion.id }, + data: { url: normalizedUrl }, + }); + console.log(`Updated PageVersion URL: ${pageVersion.url} -> ${normalizedUrl}`); + } + } + + processedPageVersionCount += pageVersions.length; + console.log(`Processed ${processedPageVersionCount} page versions`); + } + + console.log("URL normalization complete"); + } catch (error) { + console.error("Error during URL normalization:", error); + } finally { + await prisma.$disconnect(); + } +} + +normalizeUrls(); \ No newline at end of file From 44008e3c83f6d72c1f46a27a4f66a4ad13fe265a Mon Sep 17 00:00:00 2001 From: ttizze Date: Sat, 27 Jul 2024 14:20:29 +0000 Subject: [PATCH 3/4] Apply automatic changes --- web/scripts/normalize-page-urls.ts | 108 +++++++++++++++-------------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/web/scripts/normalize-page-urls.ts b/web/scripts/normalize-page-urls.ts index 47a69f16..9addf3cb 100644 --- a/web/scripts/normalize-page-urls.ts +++ b/web/scripts/normalize-page-urls.ts @@ -4,67 +4,69 @@ import { normalizeAndSanitizeUrl } from "../app/utils/normalize-and-sanitize-url const prisma = new PrismaClient(); async function normalizeUrls() { - const batchSize = 100; - let processedPageCount = 0; - let processedPageVersionCount = 0; + const batchSize = 100; + let processedPageCount = 0; + let processedPageVersionCount = 0; - try { - // Normalize Page URLs - while (true) { - const pages = await prisma.page.findMany({ - take: batchSize, - skip: processedPageCount, - select: { id: true, url: true }, - }); + try { + // Normalize Page URLs + while (true) { + const pages = await prisma.page.findMany({ + take: batchSize, + skip: processedPageCount, + select: { id: true, url: true }, + }); - if (pages.length === 0) break; + if (pages.length === 0) break; - for (const page of pages) { - const normalizedUrl = normalizeAndSanitizeUrl(page.url); - if (normalizedUrl !== page.url) { - await prisma.page.update({ - where: { id: page.id }, - data: { url: normalizedUrl }, - }); - console.log(`Updated Page URL: ${page.url} -> ${normalizedUrl}`); - } - } + for (const page of pages) { + const normalizedUrl = normalizeAndSanitizeUrl(page.url); + if (normalizedUrl !== page.url) { + await prisma.page.update({ + where: { id: page.id }, + data: { url: normalizedUrl }, + }); + console.log(`Updated Page URL: ${page.url} -> ${normalizedUrl}`); + } + } - processedPageCount += pages.length; - console.log(`Processed ${processedPageCount} pages`); - } + processedPageCount += pages.length; + console.log(`Processed ${processedPageCount} pages`); + } - // Normalize PageVersion URLs - while (true) { - const pageVersions = await prisma.pageVersion.findMany({ - take: batchSize, - skip: processedPageVersionCount, - select: { id: true, url: true }, - }); + // Normalize PageVersion URLs + while (true) { + const pageVersions = await prisma.pageVersion.findMany({ + take: batchSize, + skip: processedPageVersionCount, + select: { id: true, url: true }, + }); - if (pageVersions.length === 0) break; + if (pageVersions.length === 0) break; - for (const pageVersion of pageVersions) { - const normalizedUrl = normalizeAndSanitizeUrl(pageVersion.url); - if (normalizedUrl !== pageVersion.url) { - await prisma.pageVersion.update({ - where: { id: pageVersion.id }, - data: { url: normalizedUrl }, - }); - console.log(`Updated PageVersion URL: ${pageVersion.url} -> ${normalizedUrl}`); - } - } + for (const pageVersion of pageVersions) { + const normalizedUrl = normalizeAndSanitizeUrl(pageVersion.url); + if (normalizedUrl !== pageVersion.url) { + await prisma.pageVersion.update({ + where: { id: pageVersion.id }, + data: { url: normalizedUrl }, + }); + console.log( + `Updated PageVersion URL: ${pageVersion.url} -> ${normalizedUrl}`, + ); + } + } - processedPageVersionCount += pageVersions.length; - console.log(`Processed ${processedPageVersionCount} page versions`); - } + processedPageVersionCount += pageVersions.length; + console.log(`Processed ${processedPageVersionCount} page versions`); + } - console.log("URL normalization complete"); - } catch (error) { - console.error("Error during URL normalization:", error); - } finally { - await prisma.$disconnect(); - } + console.log("URL normalization complete"); + } catch (error) { + console.error("Error during URL normalization:", error); + } finally { + await prisma.$disconnect(); + } } -normalizeUrls(); \ No newline at end of file +normalizeUrls(); From 6558bc7088d1b2770d91fcd20657661a987de7f2 Mon Sep 17 00:00:00 2001 From: tomolld Date: Sat, 27 Jul 2024 23:21:17 +0900 Subject: [PATCH 4/4] f --- web/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/web/package.json b/web/package.json index 2b7e8035..974d9152 100644 --- a/web/package.json +++ b/web/package.json @@ -5,7 +5,6 @@ "scripts": { "build": "remix vite:build", "install-and-build": "bun install && bun run build", - "deploy": "bun run build && wrangler pages deploy", "dev": "remix vite:dev", "start": "remix-serve ./build/server/index.js", "typecheck": "tsc",