WIP, tests work, but needs reflection orchestra

js/core/src/tracing.ts

@@ -42,6 +42,7 @@ const instrumentationKey = '__GENKIT_TELEMETRY_INSTRUMENTED';
  * @hidden
  */
 export async function ensureBasicTelemetryInstrumentation() {
+  console.log("enabling basic telemetry");
   if (global[instrumentationKey]) {
     return await global[instrumentationKey];
   }
@@ -54,6 +55,7 @@ export async function ensureBasicTelemetryInstrumentation() {
 export async function enableTelemetry(
   telemetryConfig: TelemetryConfig | Promise<TelemetryConfig>
 ) {
+  console.log("enabling telemetry");
   if (process.env.GENKIT_TELEMETRY_SERVER) {
     setTelemetryServerUrl(process.env.GENKIT_TELEMETRY_SERVER);
   }
@@ -102,6 +104,7 @@ export async function cleanUpTracing(): Promise<void> {
  * Creates a new SpanProcessor for exporting data to the telemetry server.
  */
 function createTelemetryServerProcessor(): SpanProcessor {
+  console.log("creating processor telemetry");
   const exporter = new TraceServerExporter();
   return isDevEnv()
     ? new SimpleSpanProcessor(exporter)
@@ -110,6 +113,7 @@ function createTelemetryServerProcessor(): SpanProcessor {
 
 /** Flush metrics if present. */
 function maybeFlushMetrics(): Promise<void> {
+  console.log("maybe flushing telemetry");
   if (nodeOtelConfig?.metricReader) {
     return nodeOtelConfig.metricReader.forceFlush();
   }
@@ -122,6 +126,7 @@ function maybeFlushMetrics(): Promise<void> {
  * @hidden
  */
 export async function flushTracing() {
+  console.log("flushing tracins");
   if (nodeOtelConfig?.spanProcessors) {
     await Promise.all(nodeOtelConfig.spanProcessors.map((p) => p.forceFlush()));
   }

js/core/src/tracing/exporter.ts

@@ -47,6 +47,7 @@ export class TraceServerExporter implements SpanExporter {
     spans: ReadableSpan[],
     resultCallback: (result: ExportResult) => void
   ): void {
+    console.log("export telemetry");
     this._sendSpans(spans, resultCallback);
   }
 
@@ -106,6 +107,7 @@ export class TraceServerExporter implements SpanExporter {
    * Exports any pending spans in exporter
    */
   forceFlush(): Promise<void> {
+    console.log("force flush 2 telemetry");
     return Promise.resolve();
   }
 
@@ -137,6 +139,7 @@ export class TraceServerExporter implements SpanExporter {
   }
 
   private async save(traceId, spans: ReadableSpan[]): Promise<void> {
+    console.log("fsaving 2 telemetry");
     if (!telemetryServerUrl) {
       logger.debug(
         `Telemetry server is not configured, trace ${traceId} not saved!`

js/core/src/tracing/instrumentation.ts

@@ -47,6 +47,7 @@ export async function newTrace<T>(
   },
   fn: (metadata: SpanMetadata, rootSpan: ApiSpan) => Promise<T>
 ) {
+  console.log("newTrace")
   registry = (registry as HasRegistry).registry
     ? (registry as HasRegistry).registry
     : (registry as Registry);
@@ -90,6 +91,8 @@ export async function runInNewSpan<T>(
   },
   fn: (metadata: SpanMetadata, otSpan: ApiSpan, isRoot: boolean) => Promise<T>
 ): Promise<T> {
+
+  console.log("runINnew span")
   await ensureBasicTelemetryInstrumentation();
   const resolvedRegistry = (registry as HasRegistry).registry
     ? (registry as HasRegistry).registry
@@ -154,6 +157,7 @@ export async function appendSpan(
   metadata: SpanMetadata,
   labels?: Record<string, string>
 ) {
+  console.log("append span")
   await ensureBasicTelemetryInstrumentation();
 
   const tracer = trace.getTracer(TRACER_NAME, TRACER_VERSION);

js/core/src/tracing/multiSpanProcessor.ts

@@ -28,10 +28,13 @@ export class MultiSpanProcessor implements SpanProcessor {
   constructor(private processors: SpanProcessor[]) {}
 
   forceFlush(): Promise<void> {
+    console.log("multispan ")
     return Promise.all(this.processors.map((p) => p.forceFlush())).then();
   }
 
   onStart(span: Span, parentContext: Context): void {
+
+  console.log("multispan ")
     this.processors.map((p) => p.onStart(span, parentContext));
   }
 

js/genkit/src/genkit.ts

@@ -840,7 +840,7 @@ export function genkit(options: GenkitOptions): Genkit {
   return new Genkit(options);
 }
 
-const shutdown = async () => {
+export const shutdown = async () => {
   logger.info('Shutting down all Genkit servers...');
   await ReflectionServer.stopAll();
   process.exit(0);
@@ -852,5 +852,6 @@ process.on('SIGINT', shutdown);
 let disableReflectionApi = false;
 
 export function __disableReflectionApi() {
+  console.log("disabling reflection");
   disableReflectionApi = true;
 }

js/genkit/src/index.ts

@@ -26,4 +26,10 @@
  * @module /
  */
 export * from './common.js';
-export { Genkit, genkit, type GenkitOptions } from './genkit.js';
+export {
+  Genkit,
+  __disableReflectionApi,
+  genkit,
+  shutdown,
+  type GenkitOptions,
+} from './genkit.js';

js/testapps/evals/jest.config.json

@@ -0,0 +1,21 @@
+{
+    "testEnvironment": "node",
+    "coveragePathIgnorePatterns": [
+      "/node_modules/"
+    ],
+    "moduleFileExtensions": ["ts", "tsx", "js", "json"],
+    "testMatch": ["**/?(*.)+(spec|test).ts"],
+    "verbose": true,
+    "preset": "ts-jest/presets/default-esm",
+    "transform": {
+        "^.+\\.tsx?$": [
+            "ts-jest",
+            {
+                "useESM": true
+            }
+        ]
+    },
+    "moduleNameMapper": {
+        "^(\\.{1,2}/.*)\\.js$": "$1"
+    }
+}

js/testapps/evals/package.json

@@ -10,7 +10,9 @@
     "build:clean": "rimraf ./lib",
     "build:watch": "tsc --watch",
     "dev": "tsx --watch src/index.ts",
-    "genkit:dev": "genkit start -- tsx --watch src/index.ts"
+    "genkit:dev": "genkit start -- tsx --watch src/index.ts",
+    "test": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --detectOpenHandles",
+    "test:watch": "cross-env NODE_OPTIONS=--experimental-vm-modules jest --detectOpenHandles --watch"
   },
   "keywords": [],
   "author": "",
@@ -28,8 +30,12 @@
     "pdfjs-dist-legacy": "^1.0.1"
   },
   "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/jest": "^29.5.12",
     "cross-env": "^7.0.3",
+    "jest": "^29.7.0",
     "rimraf": "^6.0.1",
+    "ts-jest": "^29.1.2",
     "tsx": "^4.19.2",
     "typescript": "^5.3.3"
   }

js/testapps/evals/src/genkit.ts

@@ -17,18 +17,11 @@
 import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore';
 import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator';
 import {
-  gemini15Flash,
   gemini15Pro,
   googleAI,
   textEmbeddingGecko001,
 } from '@genkit-ai/googleai';
-import { vertexAI } from '@genkit-ai/vertexai';
-import {
-  vertexAIEvaluation,
-  VertexAIEvaluationMetricType,
-} from '@genkit-ai/vertexai/evaluation';
 import { genkit } from 'genkit';
-import { langchain } from 'genkitx-langchain';
 
 // Turn off safety checks for evaluation so that the LLM as an evaluator can
 // respond appropriately to potentially harmful content without error.
@@ -70,37 +63,11 @@ export const ai = genkit({
         },
       ],
     }),
-    vertexAI({
-      location: 'us-central1',
-    }),
-    vertexAIEvaluation({
-      location: 'us-central1',
-      metrics: [
-        VertexAIEvaluationMetricType.BLEU,
-        VertexAIEvaluationMetricType.GROUNDEDNESS,
-        VertexAIEvaluationMetricType.SAFETY,
-        {
-          type: VertexAIEvaluationMetricType.ROUGE,
-          metricSpec: {
-            rougeType: 'rougeLsum',
-            useStemmer: true,
-            splitSummaries: 'true',
-          },
-        },
-      ],
-    }),
     devLocalVectorstore([
       {
         indexName: 'pdfQA',
         embedder: textEmbeddingGecko001,
       },
     ]),
-    langchain({
-      evaluators: {
-        criteria: ['coherence'],
-        labeledCriteria: ['correctness'],
-        judge: gemini15Flash,
-      },
-    }),
   ],
 });

js/testapps/evals/src/pdf-rag.ts

@@ -23,7 +23,7 @@ import { z } from 'genkit';
 import { Document } from 'genkit/retriever';
 import { chunk } from 'llm-chunk';
 import path from 'path';
-import { getDocument } from 'pdfjs-dist-legacy';
+import * as pdfjsDistLegacy from 'pdfjs-dist-legacy';
 import { ai } from './genkit.js';
 
 export const pdfChatRetriever = devLocalRetrieverRef('pdfQA');
@@ -100,11 +100,7 @@ export const simpleEcho = ai.defineFlow(
     outputSchema: z.string(),
   },
   async (i) => {
-    const llmResponse = await ai.generate({
-      model: gemini15Flash,
-      prompt: i,
-    });
-    return llmResponse.text;
+    return i;
   }
 );
 
@@ -143,7 +139,7 @@ export const indexPdf = ai.defineFlow(
 );
 
 async function extractText(filePath: string): Promise<string> {
-  let doc = await getDocument(filePath).promise;
+  let doc = await pdfjsDistLegacy.getDocument(filePath).promise;
 
   let pdfTxt = '';
   const numPages = doc.numPages;

js/testapps/evals/src/sample.test.ts

@@ -0,0 +1,28 @@
+import { __disableReflectionApi, shutdown } from 'genkit';
+import { pdfQA, simpleEcho } from './pdf-rag.js';
+
+describe('Math functions', () => {
+  beforeAll(() => {
+    __disableReflectionApi();
+  });
+
+  // afterAll(async () => {
+  //   await shutdown();
+  // });
+  // test('adds 1 + 2 to equal 3', () => {
+  //   expect(1 + 2).toBe(3);
+  // });
+
+  // test('multiplies 3 * 4 to equal 12', () => {
+  //   expect(3 * 4).toBe(12);
+  // });
+
+  test('simpleEcho', async () => {
+
+    expect(await simpleEcho('Hellozzz world')).toBe('Hellozzz world');
+  });
+
+  // test('pdfQA', async () => {
+  //   expect(await pdfQA('Can I feed milk to my cats?')).toContain('No');
+  // });
+});