Support for direct call of genaiscript from js

docs/src/content/docs/reference/cli/api.mdx

@@ -0,0 +1,51 @@
+---
+title: Node.JS API
+sidebar:
+    order: 50
+---
+
+import { Tabs, TabItem } from "@astrojs/starlight/components"
+
+This page describes how to import and use the [cli](/genaiscript/reference/cli) as an API in your Node.JS application.
+
+Assuming you have have added the cli as a dependency in your project, you can import the cli as follows:
+
+<Tabs>
+  <TabItem value="npm" label="npm">
+
+```sh
+npm install -D genaiscript
+```
+
+</TabItem>
+
+  <TabItem value="yarn" label="yarn">
+
+```sh
+yarn add -D genaiscript
+```
+
+</TabItem>
+
+</Tabs>
+
+The API can be imported using imports (CommonJS is not supported) from **"genaiscript/api"**.
+
+```javascript
+import { runScript } from "genaiscript/api"
+```
+
+The imported API is a tiny-zero dependency wrapper that takes the arguments and spawns a [Node.JS worker thread](https://nodejs.org/api/worker_threads.html) to run GenAIScript.
+
+- No pollutation of the globals
+- No side effects on the process
+
+## `run`
+
+The `run` function wraps the [cli run](/genaiscript/reference/cli/run) command.
+
+```javascript
+import { run } from "genaiscript/api"
+
+const results = await run("summarize", ["myfile.txt"])
+```

docs/src/content/docs/reference/cli/index.mdx

@@ -23,11 +23,11 @@
 
 The CLI is a Node.JS package hosted on [npm](https://www.npmjs.com/package/genaiscript).
 
--   Install [Node.JS LTS](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) (Node.JS includes npm and npx).
+- Install [Node.JS LTS](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) (Node.JS includes npm and npx).
 
 ## Installation
 
--   Install locally as a `devDependency` in your project.
+- Install locally as a `devDependency` in your project.
 
 <Tabs>
   <TabItem label="npm" icon="seti:npm">
@@ -46,13 +46,13 @@
   </TabItem>
 </Tabs>
 
--   Install it globally.
+- Install it globally.
 
 ```sh "-g"
 npm install -g genaiscript
 ```
 
--   Check that your node version is at least 20._ and npm 10._ by running this command.
+- Check that your node version is at least 20._ and npm 10._ by running this command.
 
 ```sh
 node -v
@@ -77,13 +77,13 @@
 npx genaiscript ...
 ```
 
--   Add `--yes` to skip the confirmation prompt, which is useful in a CI scenario.
+- Add `--yes` to skip the confirmation prompt, which is useful in a CI scenario.
 
 ```sh "--yes"
 npx --yes genaiscript ...
 ```
 
--   Specify the version range to avoid unexpected behavior with cached installations of the CLI using npx.
+- Specify the version range to avoid unexpected behavior with cached installations of the CLI using npx.
 
 ```sh "@^1.16.0"
 npx --yes genaiscript@^1.16.0 ...
@@ -129,9 +129,13 @@
 ```sh
 npx genaiscript scripts model [script]
 ```
 
 where [script] can be a script id or a file path.
 
+## Using a the CLI as a Node.JS API
+
+The CLI can be imported and [used as an API in your Node.JS application](/genaiscript/reference/cli/api).
+
 ## Topics
 
 <DirectoryLinks directory="reference/cli" />

packages/cli/package.json

@@ -6,7 +6,12 @@
     "bin": {
         "genaiscript": "built/genaiscript.cjs"
     },
+    "exports": {
+        ".": "./built/genaiscript.cjs",
+        "./package.json": "./package.json"
+    },
     "files": [
+        "built/api.mjs",
         "built/genaiscript.cjs"
     ],
     "publisher": "Microsoft",
@@ -98,7 +103,7 @@
         "zx": "^8.2.4"
     },
     "scripts": {
-        "compile": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk && node ../../scripts/patch-cli.mjs",
+        "compile": "esbuild src/api.ts --outfile=built/api.mjs && esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk && node ../../scripts/patch-cli.mjs",
         "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk",
         "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md",
         "vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html",

packages/cli/src/api.ts

@@ -0,0 +1,42 @@
+import type { GenerationResult } from "../../core/src/generation"
+import type { PromptScriptRunOptions } from "../../core/src/server/messages"
+import { Worker } from "node:worker_threads"
+import { fileURLToPath } from "url"
+import { dirname, join } from "node:path"
+
+/**
+ * Runs a GenAIScript script with the given files and options.
+ * This function acts similarly to the `run` command in the CLI.
+ * @param scriptId script identifier or full file path
+ * @param files list of file paths to run the script on, leave empty if not needed
+ * @param options
+ * @returns
+ */
+export async function run(
+    scriptId: string,
+    files?: string[],
+    options?: Partial<PromptScriptRunOptions>
+): Promise<{
+    exitCode: number
+    result?: GenerationResult
+}> {
+    const { label } = options || {}
+    const workerData = {
+        type: "run",
+        scriptId,
+        files: files || [],
+        options,
+    }
+
+    const filename =
+        typeof __filename === "undefined"
+            // ignore esbuild warning
+            ? join(dirname(fileURLToPath(import.meta.url)), "genaiscript.cjs")
+            : __filename
+    const worker = new Worker(filename, { workerData, name: label })
+    return new Promise((resolve, reject) => {
+        worker.on("online", () => process.stderr.write(`worker: online\n`))
+        worker.on("message", resolve)
+        worker.on("error", reject)
+    })
+}

packages/cli/src/main.ts

@@ -1,13 +1,24 @@
-// Main entry point for the CLI application
-
-// Import necessary modules and functions
 import { installGlobals } from "../../core/src/globals"
 import { cli } from "./cli"
+import { workerData } from "node:worker_threads"
+import { worker } from "./worker"
+import { run } from "./api"
+import { PromptScriptRunOptions } from "../../core/src/server/messages"
+import { GenerationResult } from "../../core/src/generation"
 
-// Initialize global settings or variables for the application
-// This might include setting up global error handlers or configurations
-installGlobals()
+export { run, type PromptScriptRunOptions, type GenerationResult }
 
-// Execute the command-line interface logic
-// This function likely handles parsing input arguments and executing commands
-cli()
+// if this file is not the entry point, skip cli
+if (require.main === module) {
+    // Initialize global settings or variables for the application
+    // This might include setting up global error handlers or configurations
+    installGlobals()
+    if (workerData) {
+        // Executes a worker
+        worker()
+    } else {
+        // Execute the command-line interface logic
+        // This function likely handles parsing input arguments and executing commands
+        cli()
+    }
+}

packages/cli/src/parse.ts

@@ -146,7 +146,7 @@ export async function parseAnyToJSON(
         else if (YAML_REGEX.test(file)) data = YAMLParse(src)
         else if (XML_REGEX.test(file)) data = XMLParse(src)
         else if (MD_REGEX.test(file) || MDX_REGEX.test(file))
-            data = YAML.parse(splitMarkdown(src).frontmatter)
+            data = YAMLParse(splitMarkdown(src).frontmatter)
         else throw new Error("Unsupported file format")
     }
 

packages/cli/src/run.ts

@@ -143,7 +143,7 @@ export async function runScriptWithExitCode(
                 host.path.basename(scriptId).replace(GENAI_ANYTS_REGEX, ""),
                 `${new Date().toISOString().replace(/[:.]/g, "-")}.trace.md`
             )
-        const res = await runScript(scriptId, files, { ...options, outTrace })
+        const res = await runScriptInternal(scriptId, files, { ...options, outTrace })
         exitCode = res.exitCode
         if (
             exitCode === SUCCESS_ERROR_CODE ||
@@ -162,7 +162,7 @@ export async function runScriptWithExitCode(
     process.exit(exitCode)
 }
 
-export async function runScript(
+export async function runScriptInternal(
     scriptId: string,
     files: string[],
     options: Partial<PromptScriptRunOptions> &

packages/cli/src/server.ts

@@ -1,7 +1,7 @@
 import { WebSocketServer } from "ws"
 import { runPromptScriptTests } from "./test"
 import { PROMPTFOO_VERSION } from "./version"
-import { runScript } from "./run"
+import { runScriptInternal } from "./run"
 import { AbortSignalCancellationController } from "../../core/src/cancellation"
 import {
     SERVER_PORT,
@@ -304,7 +304,7 @@ export async function startServer(options: { port: string; apiKey?: string }) {
                             )
                         })
                         logVerbose(`run ${runId}: starting ${script}`)
-                        const runner = runScript(script, files, {
+                        const runner = runScriptInternal(script, files, {
                             ...options,
                             trace,
                             cancellationToken: canceller.token,

packages/cli/src/tsconfig.json

@@ -3,7 +3,9 @@
     "compilerOptions": {
         "types": ["node"],
         "declarationDir": "../built/types",
-        "outDir": "../built"
+        "outDir": "../built",
+        "skipLibCheck": true,
+        "emitDeclarationOnly": true
     },
     "include": [".", "../../core/src/types/*.d.ts"]
 }

packages/cli/src/worker.ts

@@ -0,0 +1,22 @@
+import { workerData, parentPort } from "node:worker_threads"
+import { runScriptInternal } from "./run"
+import { NodeHost } from "./nodehost"
+
+export async function worker() {
+    await NodeHost.install(undefined) // Install NodeHost with environment options
+
+    const { type, ...data } = workerData as {
+        type: string
+    } & object
+    switch (type) {
+        case "run": {
+            const { scriptId, files, ...options } = data as {
+                scriptId: string
+                files: string[]
+            } & object
+            const { result } = await runScriptInternal(scriptId, files, options)
+            parentPort.postMessage(result)
+            break
+        }
+    }
+}

packages/core/src/genaiscript-api-provider.mjs

@@ -4,11 +4,6 @@
  * Do not edit, auto-generated.
  *
  */
-import { promisify } from "node:util"
-import { exec } from "node:child_process"
-
-const execAsync = promisify(exec)
-
 class GenAIScriptApiProvider {
     constructor(options) {
         this.config = options.config
@@ -22,81 +17,24 @@ class GenAIScriptApiProvider {
         return this.providerId
     }
 
-    async callApi(prompt, context) {
-        const {
-            model,
-            smallModel,
-            visionModel,
-            temperature,
-            top_p,
-            cache,
-            version,
-            cli,
-            quiet,
-        } = this.config
-        const { vars, logger } = context
+    async callApi(scriptId, context) {
+        const { logger } = context
         try {
-            let files = vars.files // string or string[]
-            const testVars = vars.vars // {}
+            let files = context.vars.files // string or string[]
             if (files && !Array.isArray(files)) files = [files] // ensure array
 
-            const args = []
-            if (cli) args.push(`node`, cli)
-            else
-                args.push(
-                    `npx`,
-                    `--yes`,
-                    version ? `genaiscript@${version}` : "genaiscript"
-                )
-
-            args.push("run", prompt)
-            if (files) args.push(...files)
-            args.push("--run-retry", 2)
-            if (testVars && typeof testVars === "object") {
-                args.push("--vars")
-                for (const [key, value] of Object.entries(testVars)) {
-                    args.push(`${key}=${JSON.stringify(value)}`)
-                }
-            }
-            args.push("--json")
-            if (quiet) args.push("--quiet")
-            if (model) args.push("--model", model)
-            if (smallModel) args.push("--small-model", smallModel)
-            if (visionModel) args.push("--vision-model", visionModel)
-            if (temperature !== undefined)
-                args.push("--temperature", temperature)
-            if (top_p !== undefined) args.push("--top_p", top_p)
-            if (cache === true) args.push("--cache")
-
-            const cmd = args
-                .map((a) =>
-                    typeof a === "string" && a.includes(" ")
-                        ? JSON.stringify(a)
-                        : a
-                )
-                .join(" ")
-            logger.info(cmd)
-            let { stdout, stderr, error } = await execAsync(cmd)
-            logger.debug(stderr)
-
-            const outputText = stdout.slice(Math.max(0, stdout.indexOf("{")))
-            let output
-            try {
-                output = JSON.parse(outputText)
-                if (output.status === "error")
-                    error = output.statusText || error || "error"
-            } catch (e) {
-                error = e?.message || "error parsing genaiscript json output"
-                output = {
-                    text: outputText,
-                    error,
-                }
-            }
+            const { cli, ...options } = structuredClone(this.config)
+            options.runTries = 2
 
-            if (error) logger.error(error)
+            const testVars = context.vars.vars // {}
+            if (testVars && typeof testVars === "object")
+                options.vars = { ...(this.config.vars || []), ...testVars }
+            const api = await import(cli ?? "genaiscript")
+            const res = await api.run(scriptId, files, options)
+            logger.debug(res)
             return {
-                output,
-                error,
+                output: res,
+                error: res?.error,
             }
         } catch (e) {
             logger.error(e)