feat: implement emoji generation

package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "cac": "^6.7.14",
+    "consola": "^3.4.0",
     "farver": "^0.4.0",
     "fs-extra": "^11.3.0",
     "semver": "^7.7.1",

src/adapter/base.ts

@@ -1,32 +1,40 @@
-import type { EmojiGroup } from "../types";
-import { defineMojiAdapter } from "../adapter";
-import { slugify } from "../utils";
+import type { EmojiGroup, EmojiMetadata, EmojiShortcode, ShortcodeProvider } from "../types";
+import consola from "consola";
+import { red, yellow } from "farver/fast";
+import { defineMojiAdapter, MojisNotImplemented } from "../adapter";
+import { extractEmojiVersion, extractUnicodeVersion, slugify } from "../utils";
 import { fetchCache } from "../utils/cache";
 
 function notImplemented(adapterFn: string) {
   return async () => {
-    throw new Error(`the adapter function ${adapterFn} is not implemented`);
+    throw new MojisNotImplemented(`the adapter function ${red(adapterFn)} is not implemented`);
   };
 }
 
 export default defineMojiAdapter({
   name: "base",
   description: "base adapter",
   range: "*",
-  groups: async ({ version, force }) => {
-    if (version === "1.0" || version === "2.0" || version === "3.0") {
-      console.warn(`version ${version} does not have group data`);
-      return [];
+  metadata: async (ctx) => {
+    if (ctx.emojiVersion === "1.0" || ctx.emojiVersion === "2.0" || ctx.emojiVersion === "3.0") {
+      consola.warn(`skipping metadata for emoji version ${yellow(ctx.emojiVersion)}, as it's not supported.`);
+      return {
+        groups: [],
+        emojiMetadata: {},
+      };
     }
 
-    const groups = await fetchCache(`https://unicode.org/Public/emoji/${version}/emoji-test.txt`, {
-      cacheKey: `v${version}/metadata.json`,
+    return fetchCache(`https://unicode.org/Public/emoji/${ctx.emojiVersion}/emoji-test.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/metadata.json`,
       parser(data) {
         const lines = data.split("\n");
         let currentGroup: EmojiGroup | undefined;
 
         const groups: EmojiGroup[] = [];
 
+        // [group-subgroup][hexcode] = metadata
+        const emojiMetadata: Record<string, Record<string, EmojiMetadata>> = {};
+
         for (const line of lines) {
           if (line.trim() === "") {
             continue;
@@ -44,23 +52,123 @@ export default defineMojiAdapter({
             currentGroup = group;
 
             groups.push(group);
+
+            continue;
           } else if (line.startsWith("# subgroup:")) {
             const subgroupName = line.slice(11).trim();
 
             if (currentGroup == null) {
               throw new Error(`subgroup ${subgroupName} without group`);
             }
 
-            currentGroup.subgroups.push(subgroupName);
+            currentGroup.subgroups.push(slugify(subgroupName));
+
+            continue;
+          } else if (line.startsWith("#")) {
+            continue;
+          }
+
+          const [baseHexcode, trailingLine] = line.split(";");
+
+          if (baseHexcode == null || trailingLine == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          const [baseQualifier, comment] = trailingLine.split("#");
+
+          if (baseQualifier == null || comment == null) {
+            throw new Error(`invalid line: ${line}`);
           }
+
+          const hexcode = baseHexcode.trim().replace(/\s+/g, "-");
+          const qualifier = baseQualifier.trim();
+
+          const emojiVersion = extractEmojiVersion(comment.trim());
+          const [emoji, trimmedComment] = comment.trim().split(` E${emojiVersion} `);
+
+          const groupName = currentGroup?.slug ?? "unknown";
+          const subgroupName = currentGroup?.subgroups[currentGroup.subgroups.length - 1] ?? "unknown";
+
+          const metadataGroup = `${groupName}-${subgroupName}`;
+
+          if (emojiMetadata[metadataGroup] == null) {
+            emojiMetadata[metadataGroup] = {};
+          }
+
+          emojiMetadata[metadataGroup][hexcode] = {
+            group: groupName,
+            subgroup: subgroupName,
+            qualifier,
+            emojiVersion: emojiVersion || null,
+            unicodeVersion: extractUnicodeVersion(emojiVersion, ctx.unicodeVersion),
+            description: trimmedComment || "",
+            emoji: emoji || null,
+            hexcodes: hexcode.split("-"),
+          };
         }
 
-        return groups;
+        return {
+          groups,
+          emojiMetadata,
+        };
       },
-      bypassCache: force,
+      bypassCache: ctx.force,
     });
-
-    return groups;
   },
   sequences: notImplemented("sequences"),
+  emojis: notImplemented("emojis"),
+  variations: notImplemented("variations"),
+  unicodeNames: async (ctx) => {
+    return fetchCache(`https://unicode.org/Public/${ctx.emojiVersion}.0/ucd/UnicodeData.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/unicode-names.json`,
+      parser(data) {
+        const lines = data.split("\n");
+        const unicodeNames: Record<string, string> = {};
+
+        for (const line of lines) {
+          if (line.trim() === "" || line.startsWith("#")) {
+            continue;
+          }
+
+          const [hex, name] = line.split(";").map((col) => col.trim());
+
+          if (hex == null || name == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          unicodeNames[hex] = name;
+        }
+
+        return unicodeNames;
+      },
+      bypassCache: ctx.force,
+    });
+  },
+  async shortcodes(ctx) {
+    const providers = ctx.providers;
+
+    if (providers.length === 0) {
+      throw new Error("no shortcode providers specified");
+    }
+
+    const shortcodes: Partial<Record<ShortcodeProvider, EmojiShortcode[]>> = {};
+
+    if (this.emojis == null) {
+      throw new MojisNotImplemented("emojis");
+    }
+
+    const emojis = await this.emojis(ctx);
+
+    if (providers.includes("github")) {
+      const githubShortcodesFn = await import("../shortcode/github").then((m) => m.generateGitHubShortcodes);
+
+      shortcodes.github = await githubShortcodesFn({
+        emojis,
+        force: ctx.force,
+        version: ctx.emojiVersion,
+      });
+    }
+
+    return shortcodes;
+  },
 });

src/adapter/index.ts

@@ -1,4 +1,4 @@
-import type { EmojiGroup, EmojiSequence, EmojiVariation } from "../types";
+import type { Emoji, EmojiData, EmojiGroup, EmojiMetadata, EmojiSequence, EmojiShortcode, EmojiVariation, ShortcodeProvider } from "../types";
 import semver from "semver";
 
 export interface MojiAdapter {
@@ -22,11 +22,6 @@
    */
   extend?: string;
 
-  /**
-   * A function to generate the emoji groups for the specified version.
-   */
-  groups?: GroupFn;
-
   /**
    * A function to generate the emoji sequences for the specified version
    */
@@ -41,17 +36,31 @@
    * A function to generate emoji variations for the specified version.
    */
   variations?: EmojiVariationFn;
+
+  shortcodes?: ShortcodeFn;
+
+  metadata?: MetadataFn;
+
+  unicodeNames?: UnicodeNamesFn;
 }
 
 export interface BaseAdapterContext {
-  version: string;
+  emojiVersion: string;
+  unicodeVersion: string;
   force: boolean;
 }
 
-export type GroupFn = (ctx: BaseAdapterContext) => Promise<EmojiGroup[]>;
+export type UnicodeNamesFn = (ctx: BaseAdapterContext) => Promise<Record<string, string>>;
 export type SequenceFn = (ctx: BaseAdapterContext) => Promise<{ zwj: EmojiSequence[]; sequences: EmojiSequence[] }>;
-export type EmojiFn = (ctx: BaseAdapterContext) => Promise<any>;
+export type EmojiFn = (ctx: BaseAdapterContext) => Promise<Record<string, Emoji>>;
 export type EmojiVariationFn = (ctx: BaseAdapterContext) => Promise<EmojiVariation[]>;
+export type ShortcodeFn = (ctx: BaseAdapterContext & {
+  providers: ShortcodeProvider[];
+}) => Promise<Partial<Record<ShortcodeProvider, EmojiShortcode[]>>>;
+export type MetadataFn = (ctx: BaseAdapterContext) => Promise<{
+  groups: EmojiGroup[];
+  emojiMetadata: Record<string, Record<string, EmojiMetadata>>;
+}>;
 
 export const ADAPTERS = new Map<string, MojiAdapter>();
 
@@ -82,3 +91,10 @@
 
   return adapter;
 }
+
+export class MojisNotImplemented extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "MojisNotImplemented";
+  }
+}

src/adapter/v16.ts

@@ -1,6 +1,7 @@
-import type { EmojiSequence, EmojiVariation } from "../types";
+import type { Emoji, EmojiData, EmojiSequence, EmojiShortcode, EmojiVariation, Property, ShortcodeProvider } from "../types";
 import { defineMojiAdapter } from "../adapter";
 import { FEMALE_SIGN, MALE_SIGN } from "../constants";
+import { extractEmojiVersion, extractUnicodeVersion } from "../utils";
 import { fetchCache } from "../utils/cache";
 import { expandHexRange } from "../utils/hexcode";
 
@@ -12,12 +13,12 @@
   sequences: async (ctx) => {
     const [sequences, zwj] = await Promise.all([
       {
-        cacheKey: `v${ctx.version}/sequences.json`,
-        url: `https://unicode.org/Public/emoji/${ctx.version}/emoji-sequences.txt`,
+        cacheKey: `v${ctx.emojiVersion}/sequences.json`,
+        url: `https://unicode.org/Public/emoji/${ctx.emojiVersion}/emoji-sequences.txt`,
       },
       {
-        cacheKey: `v${ctx.version}/zwj-sequences.json`,
-        url: `https://unicode.org/Public/emoji/${ctx.version}/emoji-zwj-sequences.txt`,
+        cacheKey: `v${ctx.emojiVersion}/zwj-sequences.json`,
+        url: `https://unicode.org/Public/emoji/${ctx.emojiVersion}/emoji-zwj-sequences.txt`,
       },
     ].map(async ({ cacheKey, url }) => {
       return await fetchCache(url, {
@@ -28,7 +29,7 @@
           const sequences: EmojiSequence[] = [];
 
           for (let line of lines) {
-          // skip empty line & comments
+            // skip empty line & comments
             if (line.trim() === "" || line.startsWith("#")) {
               continue;
             }
@@ -68,11 +69,87 @@
       zwj: zwj || [],
     };
   },
-  async emojis({ version, force }) {
+  async emojis(ctx) {
+    const unicodeNames = await this.unicodeNames!(ctx);
+    const { sequences, zwj } = await this.sequences!(ctx);
+    const metadata = await this.metadata!(ctx);
+    const variations = await this.variations!(ctx);
+
+    const emojis: Record<string, Emoji> = {};
+
+    const emojiData = await fetchCache(`https://unicode.org/Public/${ctx.emojiVersion}.0/ucd/emoji/emoji-data.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/emoji-data.json`,
+      parser(data) {
+        const lines = data.split("\n");
+
+        const emojiData: Record<string, EmojiData> = {};
+
+        for (const line of lines) {
+          // skip empty line & comments
+          if (line.trim() === "" || line.startsWith("#")) {
+            continue;
+          }
+
+          const lineCommentIndex = line.indexOf("#");
+          const lineComment = lineCommentIndex !== -1 ? line.slice(lineCommentIndex + 1).trim() : "";
+
+          let [hex, property] = line.split(";").map((col) => col.trim()).slice(0, 4);
+
+          if (hex == null || property == null) {
+            throw new Error(`invalid line: ${line}`);
+          }
+
+          // remove line comment from property
+          const propertyCommentIndex = property.indexOf("#");
+          if (propertyCommentIndex !== -1) {
+            property = property.slice(0, propertyCommentIndex).trim();
+          }
+
+          if (property === "Extended_Pictographic") {
+            continue;
+          }
+
+          const expandedHex = expandHexRange(hex);
+          const emojiVersion = extractEmojiVersion(lineComment);
+
+          const emoji: EmojiData = {
+            description: lineComment,
+            hexcode: "",
+            gender: null,
+            properties: [(property as Property) || "Emoji"],
+            unicodeVersion: extractUnicodeVersion(emojiVersion, ctx.unicodeVersion),
+            emojiVersion,
+            name: unicodeNames[hex] || "",
+          };
+
+          for (const hex of expandedHex) {
+            if (emojiData[hex] != null) {
+              emojiData[hex].properties = [...new Set([...emojiData[hex].properties, ...emoji.properties])];
+            } else {
+              emojiData[hex] = {
+                ...emoji,
+                hexcode: hex.replace(/\s+/g, "-"),
+              };
+            }
+          }
+        }
+
+        return emojiData;
+      },
+      bypassCache: ctx.force,
+    });
+
+    // join names, metadata, variations, sequences, zwj
+
+    for (const [hex, data] of Object.entries(emojiData)) {
+
+    }
+
+    return {};
   },
   variations: async (ctx) => {
-    return fetchCache(`https://unicode.org/Public/${ctx.version}.0/ucd/emoji/emoji-variation-sequences.txt`, {
-      cacheKey: `v${ctx.version}/variations.json`,
+    return fetchCache(`https://unicode.org/Public/${ctx.emojiVersion}.0/ucd/emoji/emoji-variation-sequences.txt`, {
+      cacheKey: `v${ctx.emojiVersion}/variations.json`,
       parser(data) {
         const lines = data.split("\n");