Add prompt metadata to registry for function-based prompts (#1484)

Author: MichaelDoyle

js/genkit/src/genkit.ts

@@ -130,6 +130,7 @@ import {
   PromptMetadata as DotpromptPromptMetadata,
   loadPromptFolder,
   prompt,
+  toFrontmatter,
 } from '@genkit-ai/dotprompt';
 import { v4 as uuidv4 } from 'uuid';
 import { BaseEvalDataPointSchema } from './evaluator.js';
@@ -360,7 +361,16 @@ export class Genkit implements HasRegistry {
   }
 
   /**
-   * Defines and registers a function-based prompt.
+   * Defines and registers a prompt based on a function.
+   *
+   * This is an alternative to defining and importing a .prompt file, providing
+   * the most advanced control over how the final request to the model is made.
+   *
+   * @param options - Prompt metadata including model, model params,
+   * input/output schemas, etc
+   * @param fn - A function that returns a {@link GenerateRequest}. Any config
+   * parameters specified by the {@link GenerateRequest} will take precedence
+   * over any parameters specified by `options`.
    *
    * ```ts
    * const hi = ai.definePrompt(
@@ -394,9 +404,13 @@ export class Genkit implements HasRegistry {
   ): ExecutablePrompt<z.infer<I>, O, CustomOptions>;
 
   /**
-   * Defines and registers a dotprompt.
+   * Defines and registers a prompt based on a template.
    *
-   * This is an alternative to defining and importing a .prompt file.
+   * This is an alternative to defining and importing a .prompt file, in
+   * situations where a static definition will not suffice.
+   *
+   * @param options - The first input number
+   * @param fn - The second input number
    *
    * ```ts
    * const hi = ai.definePrompt(
@@ -427,10 +441,7 @@ export class Genkit implements HasRegistry {
     O extends z.ZodTypeAny = z.ZodTypeAny,
     CustomOptions extends z.ZodTypeAny = z.ZodTypeAny,
   >(
-    options: PromptMetadata<I, CustomOptions> & {
-      /** The name of the prompt. */
-      name: string;
-    },
+    options: PromptMetadata<I, CustomOptions>,
     templateOrFn: string | PromptFn<I>
   ): ExecutablePrompt<z.infer<I>, O, CustomOptions> {
     if (!options.name) {
@@ -458,9 +469,17 @@ export class Genkit implements HasRegistry {
         this.registry,
         {
           name: options.name!,
+          description: options.description,
           inputJsonSchema: options.input?.jsonSchema,
           inputSchema: options.input?.schema,
-          description: options.description,
+          metadata: {
+            type: 'prompt',
+            // TODO: As a stop-gap, we are using the dotprompt interpretation of
+            // the "prompt metadata", which is roughly the same as the dotprompt
+            // frontmatter schema. This should be inverted, such that Genkit
+            // defines the metadata spec and registered dotprompts conform.
+            prompt: toFrontmatter(options),
+          },
         },
         async (input: z.infer<I>) => {
           const response = await (templateOrFn as PromptFn<I>)(input);

js/genkit/tests/prompts_test.ts

@@ -18,7 +18,7 @@ import { modelRef } from '@genkit-ai/ai/model';
 import assert from 'node:assert';
 import { beforeEach, describe, it } from 'node:test';
 import { Genkit, genkit } from '../src/genkit';
-import { z } from '../src/index';
+import { PromptAction, z } from '../src/index';
 import {
   ProgrammableModel,
   defineEchoModel,
@@ -509,7 +509,7 @@ describe('definePrompt', () => {
       defineEchoModel(ai);
     });
 
-    it('calls dotprompt with default model', async () => {
+    it('calls prompt with default model', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -532,7 +532,7 @@ describe('definePrompt', () => {
       assert.strictEqual(response.text, 'Echo: hi Genkit; config: {}');
     });
 
-    it('calls dotprompt with default model with config', async () => {
+    it('calls prompt with default model with config', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -561,7 +561,7 @@ describe('definePrompt', () => {
       );
     });
 
-    it('calls dotprompt with default model via retrieved prompt', async () => {
+    it('calls prompt with default model via retrieved prompt', async () => {
       ai.definePrompt(
         {
           name: 'hi',
@@ -599,7 +599,7 @@ describe('definePrompt', () => {
       defineEchoModel(ai);
     });
 
-    it('calls dotprompt with default model', async () => {
+    it('calls prompt with default model', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -622,7 +622,7 @@ describe('definePrompt', () => {
       assert.strictEqual(response.text, 'Echo: hi Genkit; config: {}');
     });
 
-    it('streams dotprompt with default model', async () => {
+    it('streams prompt with default model', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -667,7 +667,7 @@ describe('definePrompt', () => {
       defineEchoModel(ai);
     });
 
-    it('calls dotprompt with default model', async () => {
+    it('calls prompt with default model', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -691,7 +691,7 @@ describe('definePrompt', () => {
       assert.strictEqual(response.text, 'Echo: hi Genkit; config: {}');
     });
 
-    it('calls dotprompt with default model with config', async () => {
+    it('calls prompt with default model with config', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -721,7 +721,7 @@ describe('definePrompt', () => {
       );
     });
 
-    it('calls dotprompt with default model with call site config', async () => {
+    it('calls prompt with default model with call site config', async () => {
       const hi = ai.definePrompt(
         {
           name: 'hi',
@@ -870,6 +870,57 @@ describe('prompt', () => {
     assert.strictEqual(text, 'Echo: hi banana; config: {"temperature":11}');
   });
 
+  it('includes metadata for functional prompts', async () => {
+    ai.definePrompt(
+      {
+        name: 'hi',
+        model: 'echoModel',
+        input: {
+          schema: z.object({
+            name: z.string(),
+          }),
+        },
+        config: {
+          temperature: 0.13,
+        },
+      },
+      async (input) => {
+        return {
+          messages: [],
+          config: {
+            temperature: 11,
+          },
+        };
+      }
+    );
+    const testPrompt: PromptAction =
+      await ai.registry.lookupAction('/prompt/hi');
+
+    assert.deepStrictEqual(testPrompt.__action.metadata, {
+      type: 'prompt',
+      prompt: {
+        name: 'hi',
+        model: 'echoModel',
+        config: {
+          temperature: 0.13,
+        },
+        input: {
+          schema: {
+            type: 'object',
+            properties: {
+              name: {
+                type: 'string',
+              },
+            },
+            required: ['name'],
+            additionalProperties: true,
+            $schema: 'http://json-schema.org/draft-07/schema#',
+          },
+        },
+      },
+    });
+  });
+
   it('passes in output options to the model', async () => {
     const hi = ai.definePrompt(
       {

js/plugins/dotprompt/src/index.ts

@@ -17,6 +17,7 @@
 import { Registry } from '@genkit-ai/core/registry';
 import { readFileSync } from 'fs';
 import { basename } from 'path';
+import { toFrontmatter } from './metadata.js';
 import {
   defineDotprompt,
   Dotprompt,
@@ -31,6 +32,7 @@ export {
   defineDotprompt,
   Dotprompt,
   loadPromptFolder,
+  toFrontmatter,
   type PromptGenerateOptions,
 };