Fix support for yaml dictionary files

Author: yanthomasdev

examples/starlight/lunaria.config.json

@@ -17,6 +17,14 @@ export default defineConfig({
 			},
 			type: 'universal',
 		},
+		{
+			include: ['src/content/i18n/en.yml'],
+			pattern: {
+				source: 'src/content/i18n/@lang.yml',
+				locales: 'src/content/i18n/@lang.yml',
+			},
+			type: 'dictionary',
+		},
 	],
 	tracking: {
 		localizableProperty: 'i18nReady',
@@ -29,5 +37,4 @@ export default defineConfig({
 			'i18nIgnore',
 		],
 	},
-	external: true,
 });

examples/starlight/lunaria.config.ts

@@ -0,0 +1,2 @@
+one: test
+second: test

examples/starlight/src/content/i18n/en.yml

@@ -0,0 +1 @@
+one: teste

examples/starlight/src/content/i18n/pt.yml

@@ -5,6 +5,6 @@ const lunaria = await createLunaria({
 	force: true,
 	logLevel: 'debug',
 });
-const status = await lunaria.getFullStatus();
-console.info(status);
+const status = await lunaria.getFileStatus('src/content/i18n/en.yml');
+console.info(status.localizations[0].missingKeys);
 console.timeEnd('Lunaria benchmark');

examples/starlight/test.js

@@ -24,6 +24,10 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./utils": {
+      "types": "./dist/utils/index.d.ts",
+      "default": "./dist/utils/index.js"
+    },
     "./config": {
       "types": "./dist/config/index.d.ts",
       "default": "./dist/config/index.js"
@@ -43,6 +47,7 @@
   "dependencies": {
     "consola": "^3.2.3",
     "jiti": "2.3.3",
+    "js-yaml": "^4.1.0",
     "neotraverse": "^0.6.18",
     "path-to-regexp": "6.3.0",
     "picomatch": "^4.0.2",
@@ -52,6 +57,7 @@
     "zod": "^3.23.8"
   },
   "devDependencies": {
+    "@types/js-yaml": "^4.0.9",
     "@types/node": "^22.5.4",
     "@types/picomatch": "^3.0.1"
   },

packages/core/package.json

@@ -1,6 +1,6 @@
 import { resolve } from 'node:path';
 import { ConfigNotFound, ConfigValidationError } from '../errors/errors.js';
-import { moduleLoader } from '../files/loaders.js';
+import { loadModule } from '../files/loaders.js';
 import { LunariaPreSetupSchema } from '../integrations/schema.js';
 import type { CompleteLunariaUserConfig } from '../integrations/types.js';
 import { exists, parseWithFriendlyErrors } from '../utils/utils.js';
@@ -38,7 +38,7 @@ export async function loadConfig() {
 		throw path;
 	}
 
-	const mod = await moduleLoader(path);
+	const mod = await loadModule(path);
 	if (mod instanceof Error) {
 		throw mod;
 	}

packages/core/src/config/config.ts

@@ -61,7 +61,6 @@ export const SetupOptionsSchema = z.object({
 				}
 			>
 	>,
-	fileLoader: z.function(z.tuple([z.string()]), z.any()),
 });
 
 const LunariaIntegrationSchema = z.object({

packages/core/src/config/schema.ts

@@ -28,8 +28,8 @@ export const FailedToLoadModule = {
 export const InvalidFilesPattern = {
 	name: 'InvalidFilesPattern',
 	title: 'Invalid `files` pattern was found.',
-	message: (pattern: string, parameter: string) =>
-		`The file pattern \`${pattern}\` is missing the \`${parameter}\` parameter. Add it to your pattern string.`,
+	message: (pattern: string) =>
+		`The file pattern \`${pattern}\` is missing a valid path parameter. Be sure to add at least one to your pattern string.`,
 } satisfies ErrorContext;
 
 export const FileConfigNotFound = {
@@ -46,11 +46,18 @@ export const UncommittedFileFound = {
 		`The file \`${path}\` is being tracked but no commits have been found. Ensure all tracked files in your working branch are committed before running Lunaria.`,
 } satisfies ErrorContext;
 
-export const InvalidDictionaryFormat = {
-	name: 'InvalidDictionaryFormat',
-	title: 'A file with an invalid dictionary format was found.',
+export const InvalidDictionaryStructure = {
+	name: 'InvalidDictionaryStructure',
+	title: 'A file with an invalid dictionary structure was found.',
 	message: (path: string) =>
-		`The \`type: "dictionary"\` file \`${path}\` has an invalid format. Dictionaries are expected to be a recursive Record of string keys and values. Alternatively, you can track this file without key completion checking by setting it to \`type: "universal"\` instead.`,
+		`The \`type: "dictionary"\` file \`${path}\` has an invalid structure. Dictionaries are expected to be a recursive Record of string keys and values. Alternatively, you can track this file without key completion checking by setting it to \`type: "universal"\` instead.`,
+} satisfies ErrorContext;
+
+export const UnsupportedDictionaryFileFormat = {
+	name: 'UnsupportedDictionaryFileFormat',
+	title: 'An unsupported file format was found.',
+	message: (file: string) =>
+		`The file \`${file}\` has an unsupported file format. Dictionaries can be Markdown/MDX/Markdoc, JSON, or JavaScript/TypeScript modules. Use one of these file formats or instead track this file without key completion checking by setting it to \`type: "universal"\` instead.`,
 } satisfies ErrorContext;
 
 export const UnsupportedIntegrationSelfUpdate = {

packages/core/src/errors/errors.ts

@@ -1,13 +1,16 @@
 import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { createJiti } from 'jiti';
+import yaml from 'js-yaml';
 import { parse } from 'ultramatter';
 import { FailedToLoadModule } from '../errors/errors.js';
 
+// TODO: Consider moving this out of the loaders file and directly into the public utils API.
+
 /** Regex to match ESM and CJS JavaScript/TypeScript files. */
 export const moduleFileRe = /\.(c|m)?(ts|js)$/;
 /** Loader for JavaScript/TypeScript modules (CJS, ESM). */
-export async function moduleLoader(path: string) {
+export async function loadModule(path: string) {
 	const resolvedPath = resolve(path);
 	const jiti = createJiti(import.meta.url);
 
@@ -19,9 +22,9 @@ export async function moduleLoader(path: string) {
 }
 
 /** Regex to match files that support frontmatter. */
-export const fileSupportsFrontmatterRe = /\.(yml|md|markdown|mdx|mdoc)$/;
-/** Loader for frontmatter in `.yml`, `.md`, `.markdown`, `.mdx`, `.mdoc`. */
-export async function frontmatterLoader(path: string) {
+export const frontmatterFileRe = /\.(md|markdown|mdx|mdoc)$/;
+/** Loader for frontmatter in `.md`, `.markdown`, `.mdx`, `.mdoc`. */
+export async function loadFrontmatter(path: string) {
 	const resolvedPath = resolve(path);
 
 	try {
@@ -37,7 +40,7 @@ export async function frontmatterLoader(path: string) {
 /** Regex to match JSON files. */
 export const jsonFileRe = /\.json$/;
 /** Loader for JSON files. */
-export async function jsonLoader(path: string) {
+export async function loadJSON(path: string) {
 	const resolvedPath = resolve(path);
 
 	try {
@@ -48,9 +51,15 @@ export async function jsonLoader(path: string) {
 	}
 }
 
-/** Loader for JS/TS modules, JSON, and frontmatter.  */
-export async function fileLoader(path: string) {
-	if (moduleFileRe.test(path)) return await moduleLoader(path);
-	if (fileSupportsFrontmatterRe.test(path)) return await frontmatterLoader(path);
-	if (jsonFileRe.test(path)) return jsonLoader(path);
+export const yamlFileRe = /\.(yml|yaml)$/;
+/** Loader for YAML files. */
+export async function loadYAML(path: string) {
+	const resolvedPath = resolve(path);
+
+	try {
+		const file = await readFile(resolvedPath, 'utf-8');
+		return yaml.load(file);
+	} catch (e) {
+		if (e instanceof Error) return e;
+	}
 }

packages/core/src/files/loaders.ts

@@ -183,10 +183,10 @@ class Lunaria {
 						new Date(latestSourceChanges.latestTrackedChange.date) >
 						new Date(latestLocaleChanges.latestTrackedChange.date);
 
-					const entryTypeData = () => {
+					const entryTypeData = async () => {
 						if (fileConfig.type === 'dictionary') {
 							try {
-								const missingKeys = getDictionaryCompletion(
+								const missingKeys = await getDictionaryCompletion(
 									fileConfig.optionalKeys,
 									externalSafePath(external, this.#cwd, sourcePath),
 									externalSafePath(external, this.#cwd, localizedPath),
@@ -210,7 +210,7 @@ class Lunaria {
 						path: localizedPath,
 						git: latestLocaleChanges,
 						status: isOutdated ? 'outdated' : 'up-to-date',
-						...entryTypeData(),
+						...(await entryTypeData()),
 					};
 				}),
 			),

packages/core/src/index.ts

@@ -2,7 +2,6 @@ import type { ConsolaInstance } from 'consola';
 import { validateFinalConfig, validateInitialConfig } from '../config/config.js';
 import type { LunariaUserConfig } from '../config/types.js';
 import { UnsupportedIntegrationSelfUpdate } from '../errors/errors.js';
-import { fileLoader } from '../files/loaders.js';
 import type { CompleteLunariaUserConfig } from './types.js';
 
 export async function runSetupHook(config: LunariaUserConfig, logger: ConsolaInstance) {
@@ -37,7 +36,6 @@ export async function runSetupHook(config: LunariaUserConfig, logger: ConsolaIns
 				});
 			},
 			logger,
-			fileLoader,
 		});
 	}
 

packages/core/src/integrations/integrations.ts

@@ -8,7 +8,6 @@ export interface LunariaIntegration {
 			config: LunariaUserConfig;
 			updateConfig: (config: Partial<LunariaUserConfig>) => void;
 			logger: ConsolaInstance;
-			fileLoader: (path: string) => Promise<unknown>;
 		}) => void | Promise<void>;
 	};
 }

packages/core/src/integrations/types.ts

@@ -66,6 +66,9 @@ export class LunariaGitInstance {
 		};
 	}
 
+	// TODO: Using an external repo seems to introduce some sort of performance gains, this should be tested to ensure
+	// its not a bug e.g. not being able to read certain files or the git history not being complete and missing commits
+	// that are necessary for the status to be accurate.
 	async handleExternalRepository() {
 		const { cloneDir, repository } = this.#config;
 		const { name, hosting, rootDir } = repository;

packages/core/src/status/git.ts

@@ -1,17 +1,26 @@
 import { Traverse } from 'neotraverse/modern';
 import type { OptionalKeys } from '../config/types.js';
-import { InvalidDictionaryFormat } from '../errors/errors.js';
-import { fileLoader, fileSupportsFrontmatterRe, frontmatterLoader } from '../files/loaders.js';
+import { InvalidDictionaryStructure, UnsupportedDictionaryFileFormat } from '../errors/errors.js';
+import {
+	frontmatterFileRe,
+	jsonFileRe,
+	loadFrontmatter,
+	loadJSON,
+	loadModule,
+	loadYAML,
+	moduleFileRe,
+	yamlFileRe,
+} from '../files/loaders.js';
 import { DictionarySchema } from './schema.js';
 import type { Dictionary } from './types.js';
 
 export async function isFileLocalizable(path: string, localizableProperty: string | undefined) {
 	// If no localizableProperty is specified, all files are supposed to be localizable.
 	if (!localizableProperty) return true;
 	// If the file doesn't support frontmatter, it's automatically supposed to be localizable.
-	if (!fileSupportsFrontmatterRe.test(path)) return true;
+	if (!frontmatterFileRe.test(path)) return true;
 
-	const frontmatter = await frontmatterLoader(path);
+	const frontmatter = await loadFrontmatter(path);
 
 	if (frontmatter instanceof Error) return frontmatter;
 
@@ -31,27 +40,25 @@ export async function getDictionaryCompletion(
 	sourceDictPath: string,
 	localeDictPath: string,
 ) {
-	const sourceDict = await fileLoader(sourceDictPath);
-	const localeDict = await fileLoader(localeDictPath);
+	const [sourceDict, localeDict] = await loadDictionaries(sourceDictPath, localeDictPath);
 
 	if (sourceDict instanceof Error || localeDict instanceof Error) {
 		throw sourceDict instanceof Error ? sourceDict : localeDict;
 	}
 
 	const parsedSourceDict = DictionarySchema.safeParse(sourceDict);
 	if (parsedSourceDict.error) {
-		throw new Error(InvalidDictionaryFormat.message(sourceDictPath));
+		throw new Error(InvalidDictionaryStructure.message(sourceDictPath));
 	}
 
 	const parsedLocaleDict = DictionarySchema.safeParse(localeDict);
 	if (parsedLocaleDict.error) {
-		throw new Error(InvalidDictionaryFormat.message(localeDictPath));
+		throw new Error(InvalidDictionaryStructure.message(localeDictPath));
 	}
 
 	return findMissingKeys(optionalKeys, parsedSourceDict.data, parsedLocaleDict.data);
 }
 
-// TODO: Test this function.
 export function findMissingKeys(
 	optionalKeys: OptionalKeys | undefined,
 	sourceDict: Dictionary,
@@ -94,3 +101,20 @@ export function findMissingKeys(
 
 	return missingKeys;
 }
+
+// TODO: Add integration tests for this function
+function loadDictionaries(sourcePath: string, localePath: string) {
+	if (moduleFileRe.test(sourcePath)) {
+		return Promise.all([loadModule(sourcePath), loadModule(localePath)]);
+	}
+
+	if (yamlFileRe.test(sourcePath)) {
+		return Promise.all([loadYAML(sourcePath), loadYAML(localePath)]);
+	}
+
+	if (jsonFileRe.test(sourcePath)) {
+		return Promise.all([loadJSON(sourcePath), loadJSON(localePath)]);
+	}
+
+	throw new Error(UnsupportedDictionaryFileFormat.message(sourcePath));
+}

packages/core/src/status/status.ts

@@ -0,0 +1,2 @@
+/** Public facing utils */
+export { loadFrontmatter, loadJSON, loadModule, loadYAML } from '../files/loaders.js';

packages/core/src/utils/index.ts

@@ -3,7 +3,7 @@ import { mkdir, stat, writeFile } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import type { z } from 'zod';
 import { errorMap } from '../errors/zod-map.js';
-import { jsonLoader } from '../files/loaders.js';
+import { loadJSON } from '../files/loaders.js';
 
 export function isRelative(path: string) {
 	return path.startsWith('./') || path.startsWith('../');
@@ -75,7 +75,7 @@ export async function createCache(dir: string, entry: string, hash: string) {
 		);
 	};
 
-	const contents = async () => await jsonLoader(path);
+	const contents = async () => await loadJSON(path);
 
 	const revalidate = async (hash: string) => {
 		if ((await contents())?.__validation !== hash) {