refactor(core): replace @langchain/core with native template literals (#1506)

Co-authored-by: lilac <[email protected]>

Author: quanru

packages/core/package.json

@@ -72,7 +72,6 @@
     "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
   },
   "dependencies": {
-    "@langchain/core": "0.3.26",
     "@midscene/shared": "workspace:*",
     "@ui-tars/action-parser": "1.2.3",
     "dayjs": "^1.11.11",
@@ -82,7 +81,7 @@
     "jsonrepair": "3.12.0",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "zod": "3.25.0",
+    "zod": "3.24.3",
     "socks-proxy-agent": "8.0.4"
   },
   "devDependencies": {

packages/core/src/ai-model/inspect.ts

@@ -140,9 +140,9 @@ export async function AiLocateElement(options: {
   const targetElementDescriptionText = extraTextFromUserPrompt(
     targetElementDescription,
   );
-  const userInstructionPrompt = await findElementPrompt.format({
-    targetElementDescription: targetElementDescriptionText,
-  });
+  const userInstructionPrompt = findElementPrompt(
+    targetElementDescriptionText,
+  );
   const systemPrompt = systemPromptToLocateElement(vlMode);
 
   let imagePayload = screenshotBase64;
@@ -282,9 +282,9 @@ export async function AiLocateSection(options: {
   const { screenshotBase64 } = context;
 
   const systemPrompt = systemPromptToLocateSection(vlMode);
-  const sectionLocatorInstructionText = await sectionLocatorInstruction.format({
-    sectionDescription: extraTextFromUserPrompt(sectionDescription),
-  });
+  const sectionLocatorInstructionText = sectionLocatorInstruction(
+    extraTextFromUserPrompt(sectionDescription),
+  );
   const msgs: AIArgs = [
     { role: 'system', content: systemPrompt },
     {
@@ -396,7 +396,7 @@ export async function AiExtractElementInfo<T>(options: {
   const systemPrompt = systemPromptToExtract();
   const { screenshotBase64 } = context;
 
-  const extractDataPromptText = await extractDataQueryPrompt(
+  const extractDataPromptText = extractDataQueryPrompt(
     options.pageDescription || '',
     dataQuery,
   );

packages/core/src/ai-model/prompt/extraction.ts

@@ -1,4 +1,3 @@
-import { PromptTemplate } from '@langchain/core/prompts';
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 
 export function systemPromptToExtract() {
@@ -87,7 +86,7 @@ By viewing the screenshot and page contents, you can extract the following data:
 `;
 }
 
-export const extractDataQueryPrompt = async (
+export const extractDataQueryPrompt = (
   pageDescription: string,
   dataQuery: string | Record<string, string>,
 ) => {
@@ -97,23 +96,16 @@ export const extractDataQueryPrompt = async (
   } else {
     dataQueryText = JSON.stringify(dataQuery, null, 2);
   }
-  const extractDataPrompt = new PromptTemplate({
-    template: `
+
+  return `
 <PageDescription>
-{pageDescription}
+${pageDescription}
 </PageDescription>
 
 <DATA_DEMAND>
-{dataQuery}
+${dataQueryText}
 </DATA_DEMAND>
-  `,
-    inputVariables: ['pageDescription', 'dataQuery'],
-  });
-
-  return await extractDataPrompt.format({
-    pageDescription,
-    dataQuery: dataQueryText,
-  });
+  `;
 };
 
 export const extractDataSchema: ResponseFormatJSONSchema = {

packages/core/src/ai-model/prompt/llm-locator.ts

@@ -1,4 +1,3 @@
-import { PromptTemplate } from '@langchain/core/prompts';
 import type { TVlModeTypes } from '@midscene/shared/env';
 import { bboxDescription } from './common';
 export function systemPromptToLocateElement(vlMode: TVlModeTypes | undefined) {
@@ -41,7 +40,5 @@ When no element is found:
 `;
 }
 
-export const findElementPrompt = new PromptTemplate({
-  template: 'Find: {targetElementDescription}',
-  inputVariables: ['targetElementDescription'],
-});
+export const findElementPrompt = (targetElementDescription: string) =>
+  `Find: ${targetElementDescription}`;

packages/core/src/ai-model/prompt/llm-section-locator.ts

@@ -1,4 +1,3 @@
-import { PromptTemplate } from '@langchain/core/prompts';
 import type { TVlModeTypes } from '@midscene/shared/env';
 import { bboxDescription } from './common';
 
@@ -40,7 +39,5 @@ If the description is "delete button on the second row with title 'Peter'", retu
 `;
 }
 
-export const sectionLocatorInstruction = new PromptTemplate({
-  template: 'Find section containing: {sectionDescription}',
-  inputVariables: ['sectionDescription'],
-});
+export const sectionLocatorInstruction = (sectionDescription: string) =>
+  `Find section containing: ${sectionDescription}`;

packages/core/tests/unit-test/prompt/prompt.test.ts

@@ -220,16 +220,16 @@ describe('extract element', () => {
     expect(prompt).toMatchSnapshot();
   });
 
-  it('extract element by extractDataPrompt', async () => {
-    const prompt = await extractDataQueryPrompt(
+  it('extract element by extractDataPrompt', () => {
+    const prompt = extractDataQueryPrompt(
       'todo title, string',
       'todo title, string',
     );
     expect(prompt).toMatchSnapshot();
   });
 
-  it('extract element by extractDataPrompt - object', async () => {
-    const prompt = await extractDataQueryPrompt('todo title, string', {
+  it('extract element by extractDataPrompt - object', () => {
+    const prompt = extractDataQueryPrompt('todo title, string', {
       foo: 'an array indicates the foo',
     });
     expect(prompt).toMatchSnapshot();