diff --git a/.github/workflows/anthropic.yml b/.github/workflows/anthropic.yml index b1836a564d..5133503ffa 100644 --- a/.github/workflows/anthropic.yml +++ b/.github/workflows/anthropic.yml @@ -33,6 +33,6 @@ jobs: env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - name: cache - run: yarn test:scripts summarize-cached --model anthropic:claude-3-5-sonnet-20240620 + run: yarn run:script summarize-cached --model anthropic:claude-3-5-sonnet-20240620 --small-model anthropic:claude-3-5-sonnet-20240620 env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/docs/src/content/docs/reference/cli/api.mdx b/docs/src/content/docs/reference/cli/api.mdx new file mode 100644 index 0000000000..bacbd52d89 --- /dev/null +++ b/docs/src/content/docs/reference/cli/api.mdx @@ -0,0 +1,64 @@ +--- +title: Node.JS API +sidebar: + order: 50 +--- + +import { Tabs, TabItem } from "@astrojs/starlight/components" + +This page describes how to import and use the [cli](/genaiscript/reference/cli) as an API in your Node.JS application. + +Assuming you have have added the cli as a dependency in your project, you can import the cli as follows: + + + + +```sh +npm install -D genaiscript +``` + + + + + +```sh +yarn add -D genaiscript +``` + + + + + +The API can be imported using imports from **"genaiscript/api"**. + +```javascript +import { runScript } from "genaiscript/api" +``` + +The imported `api.mjs` wrapper is a tiny, zero dependency loader that +spawns a [Node.JS worker thread](https://nodejs.org/api/worker_threads.html) to run GenAIScript. + +- No pollutation of the globals +- No side effects on the process + +## `run` + +The `run` function wraps the [cli run](/genaiscript/reference/cli/run) command. + +```javascript +import { run } from "genaiscript/api" + +const results = await run("summarize", ["myfile.txt"]) +``` + +### Environment variables + +You can set the environment variables for the GenAIScript process by passing an object as the `env` field in the options. By default, the worker will inherit `process.env`. + +```javascript +const results = await run("summarize", ["myfile.txt"], { + env: { + MY_ENV_VAR: "value", + }, +}) +``` diff --git a/docs/src/content/docs/reference/cli/index.mdx b/docs/src/content/docs/reference/cli/index.mdx index 835da1e6a9..45ac7892b3 100644 --- a/docs/src/content/docs/reference/cli/index.mdx +++ b/docs/src/content/docs/reference/cli/index.mdx @@ -23,11 +23,11 @@ where `--yes` skips the confirmation prompt to install the package. The CLI is a Node.JS package hosted on [npm](https://www.npmjs.com/package/genaiscript). -- Install [Node.JS LTS](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) (Node.JS includes npm and npx). +- Install [Node.JS LTS](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) (Node.JS includes npm and npx). ## Installation -- Install locally as a `devDependency` in your project. +- Install locally as a `devDependency` in your project. @@ -46,13 +46,13 @@ yarn add -D genaiscript -- Install it globally. +- Install it globally. ```sh "-g" npm install -g genaiscript ``` -- Check that your node version is at least 20._ and npm 10._ by running this command. +- Check that your node version is at least 20._ and npm 10._ by running this command. ```sh node -v @@ -77,13 +77,13 @@ system issues where the tool is not found in the path. npx genaiscript ... ``` -- Add `--yes` to skip the confirmation prompt, which is useful in a CI scenario. +- Add `--yes` to skip the confirmation prompt, which is useful in a CI scenario. ```sh "--yes" npx --yes genaiscript ... ``` -- Specify the version range to avoid unexpected behavior with cached installations of the CLI using npx. +- Specify the version range to avoid unexpected behavior with cached installations of the CLI using npx. ```sh "@^1.16.0" npx --yes genaiscript@^1.16.0 ... @@ -132,6 +132,10 @@ npx genaiscript scripts model [script] where [script] can be a script id or a file path. +## Using a the CLI as a Node.JS API + +The CLI can be imported and [used as an API in your Node.JS application](/genaiscript/reference/cli/api). + ## Topics diff --git a/packages/cli/package.json b/packages/cli/package.json index bc3384de7e..47f9cae8c6 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -6,7 +6,12 @@ "bin": { "genaiscript": "built/genaiscript.cjs" }, + "exports": { + ".": "./built/genaiscript.cjs", + "./package.json": "./package.json" + }, "files": [ + "built/api.mjs", "built/genaiscript.cjs" ], "publisher": "Microsoft", @@ -98,7 +103,7 @@ "zx": "^8.2.4" }, "scripts": { - "compile": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk && node ../../scripts/patch-cli.mjs", + "compile": "esbuild src/api.ts --outfile=built/api.mjs && esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk && node ../../scripts/patch-cli.mjs", "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk", "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md", "vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html", diff --git a/packages/cli/src/api.ts b/packages/cli/src/api.ts new file mode 100644 index 0000000000..79967d190d --- /dev/null +++ b/packages/cli/src/api.ts @@ -0,0 +1,43 @@ +import type { GenerationResult } from "../../core/src/generation" +import type { PromptScriptRunOptions } from "../../core/src/server/messages" +import { Worker } from "node:worker_threads" +import { fileURLToPath } from "url" +import { dirname, join } from "node:path" + +/** + * Runs a GenAIScript script with the given files and options. + * This function acts similarly to the `run` command in the CLI. + * @param scriptId script identifier or full file path + * @param files list of file paths to run the script on, leave empty if not needed + * @param options + * @returns + */ +export async function run( + scriptId: string, + files?: string[], + options?: Partial & { + env?: Record + } +): Promise<{ + exitCode: number + result?: GenerationResult +}> { + const { env, ...rest } = options || {} + const workerData = { + type: "run", + scriptId, + files: files || [], + options: rest, + } + + const filename = + typeof __filename === "undefined" + ? join(dirname(fileURLToPath(import.meta.url)), "genaiscript.cjs") // ignore esbuild warning + : __filename + const worker = new Worker(filename, { workerData, name: options?.label }) + return new Promise((resolve, reject) => { + worker.on("online", () => process.stderr.write(`worker: online\n`)) + worker.on("message", resolve) + worker.on("error", reject) + }) +} diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 8fb25c49f0..1f91d62db5 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,13 +1,24 @@ -// Main entry point for the CLI application - -// Import necessary modules and functions import { installGlobals } from "../../core/src/globals" import { cli } from "./cli" +import { workerData } from "node:worker_threads" +import { worker } from "./worker" +import { run } from "./api" +import { PromptScriptRunOptions } from "../../core/src/server/messages" +import { GenerationResult } from "../../core/src/generation" -// Initialize global settings or variables for the application -// This might include setting up global error handlers or configurations -installGlobals() +export { run, type PromptScriptRunOptions, type GenerationResult } -// Execute the command-line interface logic -// This function likely handles parsing input arguments and executing commands -cli() +// if this file is not the entry point, skip cli +if (require.main === module) { + // Initialize global settings or variables for the application + // This might include setting up global error handlers or configurations + installGlobals() + if (workerData) { + // Executes a worker + worker() + } else { + // Execute the command-line interface logic + // This function likely handles parsing input arguments and executing commands + cli() + } +} diff --git a/packages/cli/src/parse.ts b/packages/cli/src/parse.ts index 88911c9c70..ece7e54b56 100644 --- a/packages/cli/src/parse.ts +++ b/packages/cli/src/parse.ts @@ -146,7 +146,7 @@ export async function parseAnyToJSON( else if (YAML_REGEX.test(file)) data = YAMLParse(src) else if (XML_REGEX.test(file)) data = XMLParse(src) else if (MD_REGEX.test(file) || MDX_REGEX.test(file)) - data = YAML.parse(splitMarkdown(src).frontmatter) + data = YAMLParse(splitMarkdown(src).frontmatter) else throw new Error("Unsupported file format") } diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index 1b68692d60..a7c739f09d 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -143,7 +143,7 @@ export async function runScriptWithExitCode( host.path.basename(scriptId).replace(GENAI_ANYTS_REGEX, ""), `${new Date().toISOString().replace(/[:.]/g, "-")}.trace.md` ) - const res = await runScript(scriptId, files, { ...options, outTrace }) + const res = await runScriptInternal(scriptId, files, { ...options, outTrace }) exitCode = res.exitCode if ( exitCode === SUCCESS_ERROR_CODE || @@ -162,7 +162,7 @@ export async function runScriptWithExitCode( process.exit(exitCode) } -export async function runScript( +export async function runScriptInternal( scriptId: string, files: string[], options: Partial & diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts index eb664c6788..befc039362 100644 --- a/packages/cli/src/server.ts +++ b/packages/cli/src/server.ts @@ -1,7 +1,7 @@ import { WebSocketServer } from "ws" import { runPromptScriptTests } from "./test" import { PROMPTFOO_VERSION } from "./version" -import { runScript } from "./run" +import { runScriptInternal } from "./run" import { AbortSignalCancellationController } from "../../core/src/cancellation" import { SERVER_PORT, @@ -304,7 +304,7 @@ export async function startServer(options: { port: string; apiKey?: string }) { ) }) logVerbose(`run ${runId}: starting ${script}`) - const runner = runScript(script, files, { + const runner = runScriptInternal(script, files, { ...options, trace, cancellationToken: canceller.token, diff --git a/packages/cli/src/tsconfig.json b/packages/cli/src/tsconfig.json index 75c9b60f48..a6896629c6 100644 --- a/packages/cli/src/tsconfig.json +++ b/packages/cli/src/tsconfig.json @@ -3,7 +3,9 @@ "compilerOptions": { "types": ["node"], "declarationDir": "../built/types", - "outDir": "../built" + "outDir": "../built", + "skipLibCheck": true, + "emitDeclarationOnly": true }, "include": [".", "../../core/src/types/*.d.ts"] } diff --git a/packages/cli/src/worker.ts b/packages/cli/src/worker.ts new file mode 100644 index 0000000000..edfca60a76 --- /dev/null +++ b/packages/cli/src/worker.ts @@ -0,0 +1,22 @@ +import { workerData, parentPort } from "node:worker_threads" +import { runScriptInternal } from "./run" +import { NodeHost } from "./nodehost" + +export async function worker() { + await NodeHost.install(undefined) // Install NodeHost with environment options + + const { type, ...data } = workerData as { + type: string + } & object + switch (type) { + case "run": { + const { scriptId, files, ...options } = data as { + scriptId: string + files: string[] + } & object + const { result } = await runScriptInternal(scriptId, files, options) + parentPort.postMessage(result) + break + } + } +} diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 35f4e4c000..a4a5922287 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -81,7 +81,7 @@ export const DEFAULT_SMALL_MODEL_CANDIDATES = [ "azure_serverless:gpt-4o-mini", DEFAULT_SMALL_MODEL, "google:gemini-1.5-flash-002", - "anthropic:claude-instant-1", + "anthropic:claude-instant-1.2", "mistral:mistral-small-latest", "github:gpt-4o-mini", "client:gpt-4-mini", diff --git a/packages/core/src/genaiscript-api-provider.mjs b/packages/core/src/genaiscript-api-provider.mjs index 3c0d27f7f7..26cde1c31d 100644 --- a/packages/core/src/genaiscript-api-provider.mjs +++ b/packages/core/src/genaiscript-api-provider.mjs @@ -4,11 +4,6 @@ * Do not edit, auto-generated. * */ -import { promisify } from "node:util" -import { exec } from "node:child_process" - -const execAsync = promisify(exec) - class GenAIScriptApiProvider { constructor(options) { this.config = options.config @@ -22,81 +17,24 @@ class GenAIScriptApiProvider { return this.providerId } - async callApi(prompt, context) { - const { - model, - smallModel, - visionModel, - temperature, - top_p, - cache, - version, - cli, - quiet, - } = this.config - const { vars, logger } = context + async callApi(scriptId, context) { + const { logger } = context try { - let files = vars.files // string or string[] - const testVars = vars.vars // {} + let files = context.vars.files // string or string[] if (files && !Array.isArray(files)) files = [files] // ensure array - const args = [] - if (cli) args.push(`node`, cli) - else - args.push( - `npx`, - `--yes`, - version ? `genaiscript@${version}` : "genaiscript" - ) - - args.push("run", prompt) - if (files) args.push(...files) - args.push("--run-retry", 2) - if (testVars && typeof testVars === "object") { - args.push("--vars") - for (const [key, value] of Object.entries(testVars)) { - args.push(`${key}=${JSON.stringify(value)}`) - } - } - args.push("--json") - if (quiet) args.push("--quiet") - if (model) args.push("--model", model) - if (smallModel) args.push("--small-model", smallModel) - if (visionModel) args.push("--vision-model", visionModel) - if (temperature !== undefined) - args.push("--temperature", temperature) - if (top_p !== undefined) args.push("--top_p", top_p) - if (cache === true) args.push("--cache") - - const cmd = args - .map((a) => - typeof a === "string" && a.includes(" ") - ? JSON.stringify(a) - : a - ) - .join(" ") - logger.info(cmd) - let { stdout, stderr, error } = await execAsync(cmd) - logger.debug(stderr) - - const outputText = stdout.slice(Math.max(0, stdout.indexOf("{"))) - let output - try { - output = JSON.parse(outputText) - if (output.status === "error") - error = output.statusText || error || "error" - } catch (e) { - error = e?.message || "error parsing genaiscript json output" - output = { - text: outputText, - error, - } - } + const { cli, ...options } = structuredClone(this.config) + options.runTries = 2 - if (error) logger.error(error) + const testVars = context.vars.vars // {} + if (testVars && typeof testVars === "object") + options.vars = { ...(this.config.vars || []), ...testVars } + const api = await import(cli ?? "genaiscript") + const res = await api.run(scriptId, files, options) + logger.debug(res) return { - output, - error, + output: res, + error: res?.error, } } catch (e) { logger.error(e) diff --git a/packages/core/src/pdf.ts b/packages/core/src/pdf.ts index 920b538aff..5641083f11 100644 --- a/packages/core/src/pdf.ts +++ b/packages/core/src/pdf.ts @@ -7,11 +7,6 @@ import { serializeError } from "./error" import { logVerbose, logWarn } from "./util" import { PDF_SCALE } from "./constants" -// Declare a global type for SVGGraphics as any -declare global { - export type SVGGraphics = any -} - let standardFontDataUrl: string /** diff --git a/packages/sample/genaisrc/summarize-cached.genai.mjs b/packages/sample/genaisrc/summarize-cached.genai.mjs index c911fef5ef..07dc7713af 100644 --- a/packages/sample/genaisrc/summarize-cached.genai.mjs +++ b/packages/sample/genaisrc/summarize-cached.genai.mjs @@ -1,6 +1,5 @@ script({ - title: "summarize all files", - model: "small", + title: "summarize all files with caching", files: "src/rag/markdown.md", tests: [ { diff --git a/packages/sample/src/api.mjs b/packages/sample/src/api.mjs new file mode 100644 index 0000000000..100caa04ec --- /dev/null +++ b/packages/sample/src/api.mjs @@ -0,0 +1,7 @@ +console.log(`loading cli`) +const api = await import("../../cli/built/genaiscript.cjs") + +console.log(api) +const res = await api.run("poem") + +console.log(res)