fix: retry runtime file reads -- on slow machines the delay causes errors (#1254)

Author: pavelgj

genkit-tools/common/src/manager/manager.ts

@@ -27,7 +27,7 @@ import {
 import * as apis from '../types/apis';
 import { TraceData } from '../types/trace';
 import { logger } from '../utils/logger';
-import { checkServerHealth, findRuntimesDir } from '../utils/utils';
+import { checkServerHealth, findRuntimesDir, retriable } from '../utils/utils';
 import {
   GenkitToolsError,
   RuntimeEvent,
@@ -308,8 +308,15 @@ export class RuntimeManager {
    */
   private async handleNewRuntime(filePath: string) {
     try {
-      const content = await fs.readFile(filePath, 'utf-8');
-      const runtimeInfo = JSON.parse(content) as RuntimeInfo;
+      const { content, runtimeInfo } = await retriable(
+        async () => {
+          const content = await fs.readFile(filePath, 'utf-8');
+          const runtimeInfo = JSON.parse(content) as RuntimeInfo;
+          return { content, runtimeInfo };
+        },
+        { maxRetries: 10, delayMs: 500 }
+      );
+
       if (isValidRuntimeInfo(runtimeInfo)) {
         const fileName = path.basename(filePath);
         if (await checkServerHealth(runtimeInfo.reflectionServerUrl)) {

genkit-tools/common/src/utils/utils.ts

@@ -148,3 +148,26 @@ export async function waitUntilUnresponsive(
   }
   return false;
 }
+
+export async function retriable<T>(
+  fn: () => Promise<T>,
+  opts: { maxRetries?: number; delayMs?: number }
+): Promise<T> {
+  const maxRetries = opts.maxRetries ?? 3;
+  const delayMs = opts.delayMs ?? 0;
+
+  let attempt = 0;
+  while (true) {
+    try {
+      return await fn();
+    } catch (e) {
+      if (attempt >= maxRetries - 1) {
+        throw e;
+      }
+      if (delayMs > 0) {
+        await new Promise((r) => setTimeout(r, delayMs));
+      }
+    }
+    attempt++;
+  }
+}