feat: implement emoji generation (#4)

* feat: implement emoji and variation generation commands with error handling

* feat: enhance emoji data handling with version extraction and improved parsing logic

* feat: add shortcode generator

* feat: rename groups to metadata in MojiAdapter and update related commands

* feat: enhance emoji metadata handling with version extraction and improved structure

* feat: add unicode version to adapter context

* chore: lint

* refactor: remove errors and merge into base adapter

* feat: add unicodeNames function to fetch and parse Unicode names for emojis

* refactor: migrate to a single generate command

* feat: add consola for improved logging throughout the application

* feat: implement shortcodes functionality and update emoji data structure

* feat: restructure emoji handling to include emojiData and flatten emoji groups

* chore: dump

* feat: update emoji version handling to correctly map unicode versions and simplify sorting logic

* chore: move lockfile out of utils

* chore: update test path

* feat: add README files for Unicode Emoji and Character Database draft versions

* refactor: rename extractVersion to extractVersionFromReadme for clarity

* chore: lint

* feat: add vitest-fetch-mock for enhanced testing and setup configuration

* chore: lint

Author: luxass

package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "cac": "^6.7.14",
+    "consola": "^3.4.0",
     "farver": "^0.4.0",
     "fs-extra": "^11.3.0",
     "semver": "^7.7.1",
@@ -53,6 +54,7 @@
     "tsx": "^4.19.2",
     "typescript": "^5.7.3",
     "vitest": "^3.0.5",
+    "vitest-fetch-mock": "^0.4.3",
     "vitest-testdirs": "^2.1.1"
   },
   "pnpm": {

pnpm-lock.yaml

@@ -1,32 +1,40 @@
-import type { EmojiGroup } from "../types";
-import { defineMojiAdapter } from "../adapter";
-import { slugify } from "../utils";
+import type { Emoji, EmojiGroup, EmojiMetadata, EmojiShortcode, ShortcodeProvider } from "../types";
+import consola from "consola";
+import { red, yellow } from "farver/fast";
+import { defineMojiAdapter, MojisNotImplemented } from "../adapter";
+import { extractEmojiVersion, extractUnicodeVersion, slugify } from "../utils";
 import { fetchCache } from "../utils/cache";
 
 function notImplemented(adapterFn: string) {
   return async () => {
-    throw new Error(`the adapter function ${adapterFn} is not implemented`);
+    throw new MojisNotImplemented(`the adapter function ${red(adapterFn)} is not implemented`);
   };
 }
 
 export default defineMojiAdapter({
   name: "base",
   description: "base adapter",
   range: "*",
-  groups: async ({ version, force }) => {
-    if (version === "1.0" || version === "2.0" || version === "3.0") {
-      console.warn(`version ${version} does not have group data`);
-      return [];
+  metadata: async (ctx) => {
+    if (ctx.emojiVersion === "1.0" || ctx.emojiVersion === "2.0" || ctx.emojiVersion === "3.0") {
+      consola.warn(`skipping metadata for emoji version ${yellow(ctx.emojiVersion)}, as it's not supported.`);
+      return {
+        groups: [],
+        emojiMetadata: {},
+      };
     }
 
-    const groups = await fetchCache(`https://unicode.org/Public/emoji/${version}/emoji-test.txt`, {
-      cacheKey: `v${version}/metadata.json`,
+    return fetchCache(`https://unicode.org/Public/emoji/${ctx.emojiVersion}/emoji-test.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/metadata.json`,
       parser(data) {
         const lines = data.split("\n");
         let currentGroup: EmojiGroup | undefined;
 
         const groups: EmojiGroup[] = [];
 
+        // [group-subgroup][hexcode] = metadata
+        const emojiMetadata: Record<string, Record<string, EmojiMetadata>> = {};
+
         for (const line of lines) {
           if (line.trim() === "") {
             continue;
@@ -44,23 +52,133 @@ export default defineMojiAdapter({
             currentGroup = group;
 
             groups.push(group);
+
+            continue;
           } else if (line.startsWith("# subgroup:")) {
             const subgroupName = line.slice(11).trim();
 
             if (currentGroup == null) {
               throw new Error(`subgroup ${subgroupName} without group`);
             }
 
-            currentGroup.subgroups.push(subgroupName);
+            currentGroup.subgroups.push(slugify(subgroupName));
+
+            continue;
+          } else if (line.startsWith("#")) {
+            continue;
           }
+
+          const [baseHexcode, trailingLine] = line.split(";");
+
+          if (baseHexcode == null || trailingLine == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          const [baseQualifier, comment] = trailingLine.split("#");
+
+          if (baseQualifier == null || comment == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          const hexcode = baseHexcode.trim().replace(/\s+/g, "-");
+          const qualifier = baseQualifier.trim();
+
+          const emojiVersion = extractEmojiVersion(comment.trim());
+          const [emoji, trimmedComment] = comment.trim().split(` E${emojiVersion} `);
+
+          const groupName = currentGroup?.slug ?? "unknown";
+          const subgroupName = currentGroup?.subgroups[currentGroup.subgroups.length - 1] ?? "unknown";
+
+          const metadataGroup = `${groupName}-${subgroupName}`;
+
+          if (emojiMetadata[metadataGroup] == null) {
+            emojiMetadata[metadataGroup] = {};
+          }
+
+          emojiMetadata[metadataGroup][hexcode] = {
+            group: groupName,
+            subgroup: subgroupName,
+            qualifier,
+            emojiVersion: emojiVersion || null,
+            unicodeVersion: extractUnicodeVersion(emojiVersion, ctx.unicodeVersion),
+            description: trimmedComment || "",
+            emoji: emoji || null,
+            hexcodes: hexcode.split("-"),
+          };
         }
 
-        return groups;
+        return {
+          groups,
+          emojiMetadata,
+        };
       },
-      bypassCache: force,
+      bypassCache: ctx.force,
     });
-
-    return groups;
   },
   sequences: notImplemented("sequences"),
+  emojis: notImplemented("emojis"),
+  variations: notImplemented("variations"),
+  unicodeNames: async (ctx) => {
+    return fetchCache(`https://unicode.org/Public/${ctx.emojiVersion === "13.1" ? "13.0" : ctx.emojiVersion}.0/ucd/UnicodeData.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/unicode-names.json`,
+      parser(data) {
+        const lines = data.split("\n");
+        const unicodeNames: Record<string, string> = {};
+
+        for (const line of lines) {
+          if (line.trim() === "" || line.startsWith("#")) {
+            continue;
+          }
+
+          const [hex, name] = line.split(";").map((col) => col.trim());
+
+          if (hex == null || name == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          unicodeNames[hex] = name;
+        }
+
+        return unicodeNames;
+      },
+      bypassCache: ctx.force,
+    });
+  },
+  async shortcodes(ctx) {
+    const providers = ctx.providers;
+
+    if (providers.length === 0) {
+      throw new Error("no shortcode providers specified");
+    }
+
+    const shortcodes: Partial<Record<ShortcodeProvider, EmojiShortcode[]>> = {};
+
+    if (this.emojis == null) {
+      throw new MojisNotImplemented("emojis");
+    }
+
+    const { emojis } = await this.emojis(ctx);
+
+    const flattenedEmojis = Object.values(emojis).reduce((acc, subgroup) => {
+      for (const hexcodes of Object.values(subgroup)) {
+        for (const [hexcode, emoji] of Object.entries(hexcodes)) {
+          acc[hexcode] = emoji;
+        }
+      }
+
+      return acc;
+    }, {} as Record<string, Emoji>);
+
+    if (providers.includes("github")) {
+      const githubShortcodesFn = await import("../shortcode/github").then((m) => m.generateGitHubShortcodes);
+
+      shortcodes.github = await githubShortcodesFn({
+        emojis: flattenedEmojis,
+        force: ctx.force,
+        version: ctx.emojiVersion,
+      });
+    }
+
+    return shortcodes;
+  },
 });

src/adapter/base.ts

@@ -1,4 +1,4 @@
-import type { EmojiGroup, EmojiSequence, EmojiVariation } from "../types";
+import type { Emoji, EmojiData, EmojiGroup, EmojiMetadata, EmojiSequence, EmojiShortcode, EmojiVariation, ShortcodeProvider } from "../types";
 import semver from "semver";
 
 export interface MojiAdapter {
@@ -22,11 +22,6 @@ export interface MojiAdapter {
    */
   extend?: string;
 
-  /**
-   * A function to generate the emoji groups for the specified version.
-   */
-  groups?: GroupFn;
-
   /**
    * A function to generate the emoji sequences for the specified version
    */
@@ -41,17 +36,35 @@ export interface MojiAdapter {
    * A function to generate emoji variations for the specified version.
    */
   variations?: EmojiVariationFn;
+
+  shortcodes?: ShortcodeFn;
+
+  metadata?: MetadataFn;
+
+  unicodeNames?: UnicodeNamesFn;
 }
 
 export interface BaseAdapterContext {
-  version: string;
+  emojiVersion: string;
+  unicodeVersion: string;
   force: boolean;
 }
 
-export type GroupFn = (ctx: BaseAdapterContext) => Promise<EmojiGroup[]>;
+export type UnicodeNamesFn = (ctx: BaseAdapterContext) => Promise<Record<string, string>>;
 export type SequenceFn = (ctx: BaseAdapterContext) => Promise<{ zwj: EmojiSequence[]; sequences: EmojiSequence[] }>;
-export type EmojiFn = (ctx: BaseAdapterContext) => Promise<any>;
+export type EmojiFn = (ctx: BaseAdapterContext) => Promise<{
+  emojiData: Record<string, EmojiData>;
+  // group: subgroup: hexcode: emoji
+  emojis: Record<string, Record<string, Record<string, Emoji>>>;
+}>;
 export type EmojiVariationFn = (ctx: BaseAdapterContext) => Promise<EmojiVariation[]>;
+export type ShortcodeFn = (ctx: BaseAdapterContext & {
+  providers: ShortcodeProvider[];
+}) => Promise<Partial<Record<ShortcodeProvider, EmojiShortcode[]>>>;
+export type MetadataFn = (ctx: BaseAdapterContext) => Promise<{
+  groups: EmojiGroup[];
+  emojiMetadata: Record<string, Record<string, EmojiMetadata>>;
+}>;
 
 export const ADAPTERS = new Map<string, MojiAdapter>();
 
@@ -82,3 +95,10 @@ export function defineMojiAdapter(adapter: MojiAdapter): MojiAdapter {
 
   return adapter;
 }
+
+export class MojisNotImplemented extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "MojisNotImplemented";
+  }
+}