feat: store richtext data as HTML rather than a Prosemirror tree

core/.prettierignore

@@ -1,2 +1,3 @@
 app/c/\[communitySlug\]/developers/docs/stoplight.styles.css
-vitest-bench.local.json
+vitest-bench.local.json
+**/*.html

core/app/c/[communitySlug]/pubs/[pubId]/components/PubValues.tsx

@@ -13,8 +13,8 @@ import { CoreSchemaType } from "db/public";
 import { Button } from "ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "ui/collapsible";
 import { ChevronDown, ChevronRight } from "ui/icon";
+import { ShowMore } from "ui/show-more";
 
-import type { FileUpload } from "~/lib/fields/fileUpload";
 import { FileUploadPreview } from "~/app/components/forms/FileUpload";
 import { getPubTitle, valuesWithoutTitle } from "~/lib/pubs";
 
@@ -207,11 +207,9 @@ const PubValue = ({ value }: { value: FullProcessedPubWithForm["values"][number]
 		);
 	}
 
-	// Currently, we are only rendering string versions of fields, except for file uploads
-	// For file uploads, because Unjournal doesn't have schemaNames yet, we check the value structure
 	const fileUploadSchema = getJsonSchemaByCoreSchemaType(CoreSchemaType.FileUpload);
 	if (Value.Check(fileUploadSchema, value.value)) {
-		return <FileUploadPreview files={value.value as FileUpload} />;
+		return <FileUploadPreview files={value.value} />;
 	}
 
 	if (value.schemaName === CoreSchemaType.DateTime) {
@@ -221,6 +219,17 @@ const PubValue = ({ value }: { value: FullProcessedPubWithForm["values"][number]
 		}
 	}
 
+	if (value.schemaName === CoreSchemaType.RichText) {
+		return (
+			<ShowMore animate={false}>
+				<div
+					className="prose prose-sm"
+					dangerouslySetInnerHTML={{ __html: value.value as string }}
+				/>
+			</ShowMore>
+		);
+	}
+
 	const valueAsString = (value.value as JsonValue)?.toString() || "";
 
 	let renderedField: ReactNode = valueAsString;

core/app/components/ContextEditor/ContextEditorClient.tsx

@@ -26,7 +26,7 @@ export const ContextEditorClient = ({
 	pubId,
 	pubTypeId,
 	className,
-	initialDoc,
+	initialHtml,
 	onChange,
 	disabled,
 	hideMenu,
@@ -38,9 +38,10 @@ export const ContextEditorClient = ({
 	// Might be able to use more of this type in the future—for now, this component is a lil more stricty typed than context-editor
 } & Pick<
 	ContextEditorProps,
-	"onChange" | "initialDoc" | "className" | "disabled" | "hideMenu"
+	"onChange" | "initialHtml" | "className" | "disabled" | "hideMenu"
 >) => {
 	const runUpload = useServerAction(upload);
+
 	const getPubs = useCallback(
 		(filter: string) => {
 			return new Promise<any[]>((resolve, reject) => {
@@ -65,7 +66,7 @@ export const ContextEditorClient = ({
 				}}
 				atomRenderingComponent={ContextAtom}
 				onChange={onChange}
-				initialDoc={initialDoc}
+				initialHtml={initialHtml}
 				disabled={disabled}
 				className={className}
 				hideMenu={hideMenu}

core/app/components/forms/FileUpload.tsx

@@ -1,9 +1,16 @@
 import { Button } from "ui/button";
 import { HoverCard, HoverCardContent, HoverCardTrigger } from "ui/hover-card";
 
-import type { FileUpload } from "~/lib/fields/fileUpload";
-
-export function FileUploadPreview({ files }: { files: FileUpload }) {
+export function FileUploadPreview({
+	files,
+}: {
+	files: {
+		fileName: string;
+		fileSize: number;
+		fileType: string;
+		fileUploadUrl: string;
+	}[];
+}) {
 	return (
 		<ul>
 			{files.map((file) => {

core/app/components/forms/elements/ContextEditorElement.tsx

@@ -1,9 +1,10 @@
 "use client";
 
-import type { Node } from "prosemirror-model";
+import type { ControllerRenderProps, FieldValues } from "react-hook-form";
 
-import { useState } from "react";
+import { memo, useMemo, useState } from "react";
 import { Value } from "@sinclair/typebox/value";
+import { EMPTY_DOC } from "context-editor";
 import { docHasChanged } from "context-editor/utils";
 import { useFormContext } from "react-hook-form";
 import { richTextInputConfigSchema } from "schemas";
@@ -17,81 +18,73 @@ import { ContextEditorClient } from "../../ContextEditor/ContextEditorClient";
 import { useContextEditorContext } from "../../ContextEditor/ContextEditorContext";
 import { useFormElementToggleContext } from "../FormElementToggleContext";
 
-const EMPTY_DOC = {
-	type: "doc",
-	attrs: {
-		meta: {},
-	},
-	content: [
-		{
-			type: "paragraph",
-			attrs: {
-				id: null,
-				class: null,
-			},
-		},
-	],
-};
+const EditorFormElement = memo(
+	function EditorFormElement({
+		field,
+		label,
+		help,
+	}: {
+		field: ControllerRenderProps<FieldValues, string>;
+		label: string;
+		help?: string;
+	}) {
+		const formElementToggle = useFormElementToggleContext();
+		const { pubs, pubTypes, pubId, pubTypeId } = useContextEditorContext();
 
-const EditorFormElement = ({
-	label,
-	help,
-	onChange,
-	initialValue,
-	disabled,
-}: {
-	label: string;
-	help?: string;
-	onChange: (state: any) => void;
-	initialValue?: Node;
-	disabled?: boolean;
-}) => {
-	const { pubs, pubTypes, pubId, pubTypeId } = useContextEditorContext();
-	const [initialDoc] = useState(initialValue);
+		const f = useMemo(() => {
+			return field;
+		}, []);
 
-	if (!pubId || !pubTypeId) {
-		return null;
-	}
+		const [initialHtml] = useState(f.value);
 
-	return (
-		<FormItem>
-			<FormLabel className="flex">{label}</FormLabel>
-			<div className="w-full">
-				<FormControl>
-					<ContextEditorClient
-						pubId={pubId}
-						pubs={pubs}
-						pubTypes={pubTypes}
-						pubTypeId={pubTypeId}
-						onChange={(state) => {
-							// Control changing the state more granularly or else the dirty field will trigger on load
-							// Since we can't control the dirty state directly, even this workaround does not handle the case of
-							// if someone changes the doc but then reverts it--that will still count as dirty since react-hook-form is tracking that
-							const hasChanged = docHasChanged(initialDoc ?? EMPTY_DOC, state);
-							if (hasChanged) {
-								onChange(state);
-							}
-						}}
-						initialDoc={initialDoc}
-						disabled={disabled}
-						className="max-h-96 overflow-scroll"
-					/>
-				</FormControl>
-			</div>
-			<FormDescription>{help}</FormDescription>
-			<FormMessage />
-		</FormItem>
-	);
-};
+		if (!pubId || !pubTypeId) {
+			return <></>;
+		}
+		const disabled = !formElementToggle.isEnabled(f.name);
+
+		return (
+			<FormItem>
+				<FormLabel className="flex">{label}</FormLabel>
+				<div className="w-full">
+					<FormControl>
+						<ContextEditorClient
+							pubId={pubId}
+							pubs={pubs}
+							pubTypes={pubTypes}
+							pubTypeId={pubTypeId}
+							onChange={(state, initialDoc, initialHtml) => {
+								// Control changing the state more granularly or else the dirty field will trigger on load
+								// Since we can't control the dirty state directly, even this workaround does not handle the case of
+								// if someone changes the doc but then reverts it--that will still count as dirty since react-hook-form is tracking that
+								const hasChanged = docHasChanged(initialDoc ?? EMPTY_DOC, state);
+								if (hasChanged) {
+									f.onChange(serializeProseMirrorDoc(state.doc));
+								}
+							}}
+							initialHtml={initialHtml}
+							disabled={disabled}
+							className="max-h-96 overflow-scroll"
+						/>
+					</FormControl>
+				</div>
+				<FormDescription>{help}</FormDescription>
+				<FormMessage />
+			</FormItem>
+		);
+	},
+	(prevProps, nextProps) => {
+		// delete prevProps.field;
+		// delete nextProps.field;
+		return true;
+	}
+);
 
 export const ContextEditorElement = ({
 	slug,
 	label,
 	config,
 }: ElementProps<InputComponent.richText>) => {
 	const { control } = useFormContext();
-	const formElementToggle = useFormElementToggleContext();
-	const isEnabled = formElementToggle.isEnabled(slug);
 
 	Value.Default(richTextInputConfigSchema, config);
 	if (!Value.Check(richTextInputConfigSchema, config)) {
@@ -102,19 +95,9 @@ export const ContextEditorElement = ({
 		<FormField
 			control={control}
 			name={slug}
-			render={({ field }) => {
-				return (
-					<EditorFormElement
-						label={label}
-						help={config.help}
-						onChange={(state) => {
-							field.onChange(serializeProseMirrorDoc(state.doc));
-						}}
-						initialValue={field.value}
-						disabled={!isEnabled}
-					/>
-				);
-			}}
+			render={({ field }) => (
+				<EditorFormElement field={field} label={label} help={config.help} />
+			)}
 		/>
 	);
 };

core/app/components/pubs/PubEditor/PubEditorClient.tsx

@@ -103,6 +103,7 @@ const buildDefaultValues = (
 			if (element.schemaName === CoreSchemaType.DateTime && pubValue) {
 				defaultValues[element.slug] = new Date(pubValue as string);
 			}
+
 			// There can be multiple relations for a single slug
 			if (element.isRelation) {
 				const relatedPubValues = pubValues.filter((v) => v.fieldSlug === element.slug);
@@ -163,11 +164,17 @@ const createSchemaFromElements = (
 						return [slug, undefined];
 					}
 
-					const schema = getJsonSchemaByCoreSchemaType(schemaName, config);
+					let schema = getJsonSchemaByCoreSchemaType(schemaName, config);
 					if (!schema) {
 						return [slug, undefined];
 					}
 
+					if (schemaName === CoreSchemaType.RichText) {
+						// TODO: find a better solution for this
+						// @ts-expect-error Cant assign any to the schema, but it's fine
+						schema = Type.Any();
+					}
+
 					// Allow fields to be empty or optional. Special case for empty strings,
 					// which happens when you enter something in an input field and then delete it
 					// TODO: reevaluate whether this should be "" or undefined

core/cache-handler.mjs

@@ -23,7 +23,7 @@ function createRedisHandler({
 	revalidateTagQuerySize = 1e4,
 }) {
 	function assertClientIsReady() {
-		if (!client.status === "ready") {
+		if (client.status !== "ready") {
 			// Throwing here ensures that we immediately fall back to uncached behavior, rather than
 			// waiting for the command timeout
 			throw new Error("Redis client is not ready yet or connection is lost.");
@@ -191,6 +191,7 @@ function createRedisHandler({
 
 async function getCacheHandlerPromise() {
 	let redisClient = null;
+	console.log("im walkin here");
 	if (PHASE_PRODUCTION_BUILD !== process.env.NEXT_PHASE) {
 		try {
 			redisClient = new Redis({
@@ -237,6 +238,7 @@ async function getCacheHandlerPromise() {
 
 // Usual onCreation from @neshca/cache-handler
 CacheHandler.onCreation(() => {
+	console.log("im walkin here 2");
 	// Important - It's recommended to use global scope to ensure only one Redis connection is made
 	// This ensures only one instance get created
 	if (global.cacheHandlerConfig) {

core/lib/__tests__/validateFields.test.ts

@@ -2,7 +2,7 @@ import { expect, test } from "vitest";
 
 import { CoreSchemaType } from "db/public";
 
-import { _deprecated_validatePubValuesBySchemaName } from "~/lib/server/validateFields";
+import { validatePubValuesBySchemaName } from "~/lib/server/validateFields";
 
 const JSON_EXAMPLE = {
 	content: [
@@ -20,19 +20,17 @@ const JSON_EXAMPLE = {
 };
 
 test.each([
-	{ value: "", expected: false },
 	{ value: 1, expected: false },
 	{ value: {}, expected: false },
-	{ value: JSON_EXAMPLE, expected: true },
+	{ value: JSON_EXAMPLE, expected: `<p>Example document</p>` },
+	// "" is a valid html value
+	{ value: "", expected: "" },
 ])("validating rich text field", ({ value, expected }) => {
-	const fields = [
-		{ name: "Rich Text", slug: "community:richtext", schemaName: CoreSchemaType.RichText },
-	];
-	const values = { "community:richtext": value };
-	const result = _deprecated_validatePubValuesBySchemaName({ fields, values });
-	if (expected) {
-		expect(result).toStrictEqual({});
+	const values = [{ slug: "community:richtext", value, schemaName: CoreSchemaType.RichText }];
+	const result = validatePubValuesBySchemaName(values);
+	if (expected !== false) {
+		expect(result.results[0].value).toStrictEqual(expected);
 	} else {
-		expect(result[fields[0].slug]).toBeTruthy();
+		expect(result.errors.length).toBeGreaterThan(0);
 	}
 });

core/lib/editor/process-editor-html.ts

@@ -0,0 +1,34 @@
+import rehypeFormat from "rehype-format";
+import rehypeParse from "rehype-parse";
+import rehypeStringify from "rehype-stringify";
+import { unified } from "unified";
+
+export const processEditorHTML = (
+	html: string,
+	opts?: {
+		plugins?: any[];
+		settings?: {
+			fragment?: boolean;
+			pretty?: boolean;
+		};
+	}
+) => {
+	const processor = unified().use(rehypeParse, opts?.settings);
+
+	if (opts?.settings?.pretty) {
+		processor.use(rehypeFormat);
+	}
+	if (opts?.plugins) {
+		opts.plugins.forEach((plugin) => {
+			processor.use(plugin);
+		});
+	}
+
+	return {
+		html: async () => {
+			const file = await processor.use(rehypeStringify).process(html);
+			return String(file);
+		},
+		processor,
+	};
+};