feat(cindy-web-search): add page content extraction

.tests/cindy-web-search.test.mjs

@@ -86,22 +86,29 @@ function createHarness(options = {}) {
     JSON,
     String,
     Error,
+    URL,
   });
   assert.equal(typeof handler, 'function');
 
   return {
     networkCalls,
     cindyRequests,
     toolResults,
-    async search(args = {}) {
+    async tool(tool, args = {}) {
       await handler({
         type: 'tool-call',
-        tool: 'search_web',
+        tool,
         callId: 'call-1',
-        args: { query: 'Cindy', ...args },
+        args,
       });
       return toolResults.at(-1);
     },
+    async search(args = {}) {
+      return this.tool('search_web', { query: 'Cindy', ...args });
+    },
+    async fetchPage(args = {}) {
+      return this.tool('fetch_page', { url: 'https://example.test/article', ...args });
+    },
   };
 }
 
@@ -183,23 +190,324 @@ function createSettingsHarness(options = {}) {
 }
 
 test('manifest declares Cindy Web Search and keeps BYO providers explicit', () => {
-  assert.equal(manifest.version, '1.3.2');
+  assert.equal(manifest.version, '1.4.0');
   assert.equal(manifest.minCindyVersion, '0.1.37');
   assert.deepEqual(manifest.cindy, { search: ['web'] });
   assert.ok(manifest.slots.includes('cindy'));
   assert.deepEqual(manifest.setup, { requires: [] });
-  const provider = manifest.tools[0].parameters.properties.provider;
+  const searchTool = manifest.tools.find((tool) => tool.name === 'search_web');
+  const fetchTool = manifest.tools.find((tool) => tool.name === 'fetch_page');
+  assert.ok(searchTool);
+  assert.ok(fetchTool);
+  const provider = searchTool.parameters.properties.provider;
   assert.deepEqual(provider.enum, ['cindy', 'brave', 'tavily']);
   assert.equal(provider.enum.includes('auto'), false);
-  const query = manifest.tools[0].parameters.properties.query;
+  const query = searchTool.parameters.properties.query;
   assert.equal(query.maxLength, 2000);
-  assert.match(manifest.tools[0].description, /2000/);
+  assert.match(searchTool.description, /2000/);
   assert.match(query.description, /2000/);
-  for (const locale of locales) {
+  assert.equal(fetchTool.parameters.properties.url.maxLength, 2048);
+  assert.deepEqual(fetchTool.parameters.properties.extract_depth.enum, ['basic', 'advanced']);
+  assert.match(fetchTool.description, /50000/);
+  assert.match(fetchTool.description, /不可信/);
+  assert.match(fetchTool.description, /响应过大/);
+  assert.match(fetchTool.description, /浏览器登录/);
+  assert.match(fetchTool.description, /执行页面脚本/);
+  assert.match(manifest.whenToUse, /任意公开 HTTP\(S\) 页面/);
+  assert.match(manifest.whenToUse, /包括但不限于搜索结果页/);
+  const localeRoutingContracts = [
+    [/任意公开 HTTP\(S\) 页面/, /包括但不限于搜索结果页/],
+    [/any public HTTP\(S\) page/, /including but not limited to search results/],
+    [/任意の公開 HTTP\(S\) ページ/, /検索結果ページを含む/],
+    [/모든 공개 HTTP\(S\) 페이지/, /검색 결과 페이지를 포함한/],
+  ];
+  for (const [index, locale] of locales.entries()) {
     assert.match(locale.tools.search_web.description, /2000/);
+    assert.match(locale.tools.fetch_page.description, /50000/);
+    assert.match(locale.whenToUse, localeRoutingContracts[index][0]);
+    assert.match(locale.whenToUse, localeRoutingContracts[index][1]);
+  }
+});
+
+test('fetch_page calls Tavily Extract without handling credentials in the sandbox', async () => {
+  const harness = createHarness({
+    networkResult(request) {
+      assert.equal(request.url, 'https://api.tavily.com/extract');
+      assert.equal(request.method, 'POST');
+      assert.equal(request.headers['Content-Type'], 'application/json');
+      assert.equal(request.headers.Accept, 'application/json');
+      assert.equal(request.timeoutMs, 55000);
+      assert.equal(request.callId, 'call-1');
+      assert.equal('Authorization' in request.headers, false);
+      assert.deepEqual(JSON.parse(request.body), {
+        urls: 'https://example.test/article',
+        extract_depth: 'basic',
+        include_images: false,
+        include_favicon: false,
+        format: 'markdown',
+        timeout: 20,
+      });
+      return {
+        ok: true,
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          results: [{ url: 'https://example.test/article', raw_content: '# Article\n\nBody' }],
+          failed_results: [],
+        }),
+      };
+    },
+  });
+
+  const result = await harness.fetchPage();
+
+  assert.equal(harness.networkCalls.length, 1);
+  assert.equal(harness.cindyRequests.length, 0);
+  assert.equal(result.ok, true);
+  assert.deepEqual(JSON.parse(JSON.stringify(result.result)), {
+    provider: 'tavily',
+    url: 'https://example.test/article',
+    content: '# Article\n\nBody',
+    format: 'markdown',
+    extract_depth: 'basic',
+    content_chars: 15,
+    truncated: false,
+    content_is_untrusted: true,
+  });
+});
+
+test('fetch_page supports advanced extraction and marks content truncation explicitly', async () => {
+  const content = 'x'.repeat(50001);
+  const harness = createHarness({
+    networkResult(request) {
+      const body = JSON.parse(request.body);
+      assert.equal(body.extract_depth, 'advanced');
+      assert.equal(body.timeout, 45);
+      return {
+        ok: true,
+        status: 200,
+        headers: {},
+        body: JSON.stringify({
+          results: [{ url: 'https://example.test/dynamic', raw_content: content }],
+          failed_results: [],
+        }),
+      };
+    },
+  });
+
+  const result = await harness.fetchPage({
+    url: 'https://example.test/dynamic',
+    extract_depth: 'advanced',
+  });
+
+  assert.equal(result.ok, true);
+  assert.equal(result.result.extract_depth, 'advanced');
+  assert.equal(result.result.content.length, 50000);
+  assert.equal(result.result.content_chars, 50001);
+  assert.equal(result.result.truncated, true);
+});
+
+test('fetch_page accepts uppercase characters in valid public URLs', async (t) => {
+  const cases = [
+    ['https://example.test/Article', 'https://example.test/Article'],
+    ['https://example.test/?Token=ABC', 'https://example.test/?Token=ABC'],
+    ['HTTPS://EXAMPLE.TEST/Article', 'https://example.test/Article'],
+  ];
+  for (const [url, normalizedUrl] of cases) {
+    await t.test(url, async () => {
+      const harness = createHarness({
+        networkResult(request) {
+          assert.equal(JSON.parse(request.body).urls, normalizedUrl);
+          return {
+            ok: true,
+            status: 200,
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({
+              results: [{ url: normalizedUrl, raw_content: '# Article' }],
+              failed_results: [],
+            }),
+          };
+        },
+      });
+      const result = await harness.fetchPage({ url });
+      assert.equal(result.ok, true);
+      assert.equal(result.result.url, normalizedUrl);
+      assert.equal(harness.networkCalls.length, 1);
+    });
+  }
+});
+
+test('fetch_page strips browser-local URL fragments before contacting Tavily', async () => {
+  const harness = createHarness({
+    networkResult(request) {
+      const body = JSON.parse(request.body);
+      assert.equal(body.urls, 'https://example.test/callback?Token=ABC');
+      assert.doesNotMatch(request.body, /access_token|SECRET/);
+      return {
+        ok: true,
+        status: 200,
+        headers: { 'content-type': 'application/json' },
+        body: JSON.stringify({
+          results: [{ url: body.urls, raw_content: '# Callback' }],
+          failed_results: [],
+        }),
+      };
+    },
+  });
+
+  const result = await harness.fetchPage({
+    url: 'https://example.test/callback?Token=ABC#access_token=SECRET',
+  });
+
+  assert.equal(result.ok, true);
+  assert.equal(result.result.url, 'https://example.test/callback?Token=ABC');
+  assert.equal(harness.networkCalls.length, 1);
+});
+
+test('fetch_page rejects invalid URLs before any network request', async (t) => {
+  const invalidUrls = [
+    '',
+    '/relative',
+    'file:///etc/passwd',
+    'ftp://example.test/file',
+    'https://user:pass@example.test/',
+    'https:\\example.test\\article',
+    'https://example.test/\u0000article',
+    'https://example.test/\tarticle',
+    'https://example.test/\narticle',
+    'https://example.test/\u007farticle',
+    ' https://example.test/article',
+    'http://localhost/admin',
+    'http://127.0.0.1/admin',
+    'http://10.0.0.1/admin',
+    'http://169.254.169.254/latest/meta-data/',
+    'http://192.168.1.1/admin',
+    'http://[::1]/admin',
+    'http://[fd00::1]/admin',
+    'http://[::ffff:127.0.0.1]/admin',
+    `https://example.test/${'x'.repeat(2048)}`,
+  ];
+  for (const url of invalidUrls) {
+    await t.test(JSON.stringify(url).slice(0, 80), async () => {
+      const harness = createHarness();
+      const result = await harness.fetchPage({ url });
+      assert.equal(result.ok, false);
+      assert.match(result.message, /HTTP\(S\)/);
+      assert.equal(harness.networkCalls.length, 0);
+    });
   }
 });
 
+test('fetch_page returns actionable errors without exposing upstream response bodies', async (t) => {
+  const cases = [
+    [401, /API Key/],
+    [403, /权限|账户/],
+    [429, /频繁|限流/],
+    [432, /额度|账户/],
+    [433, /额度|账户/],
+    [500, /暂时不可用/],
+  ];
+  for (const [status, expected] of cases) {
+    await t.test(String(status), async () => {
+      const harness = createHarness({
+        networkResult() {
+          return {
+            ok: true,
+            status,
+            headers: {},
+            body: 'sensitive upstream response',
+          };
+        },
+      });
+      const result = await harness.fetchPage();
+      assert.equal(result.ok, false);
+      assert.match(result.message, expected);
+      assert.doesNotMatch(result.message, /sensitive/);
+    });
+  }
+});
+
+test('fetch_page fails closed on transport and malformed provider responses', async (t) => {
+  const cases = [
+    {
+      name: 'missing key',
+      response: { ok: false, message: 'secret not configured: sensitive-name' },
+      expected: /API Key 未配置/,
+    },
+    {
+      name: 'host truncation',
+      response: { ok: true, status: 200, headers: {}, body: '{', truncated: true },
+      expected: /数据过大/,
+    },
+    {
+      name: 'oversized response',
+      response: { ok: true, status: 200, headers: {}, body: 'x'.repeat(1000001) },
+      expected: /数据过大/,
+    },
+    {
+      name: 'invalid json',
+      response: { ok: true, status: 200, headers: {}, body: '<html>upstream</html>' },
+      expected: /无法解析/,
+    },
+    {
+      name: 'failed result',
+      response: {
+        ok: true,
+        status: 200,
+        headers: {},
+        body: JSON.stringify({ results: [], failed_results: [{ url: 'https://example.test/article' }] }),
+      },
+      expected: /无法读取/,
+    },
+    {
+      name: 'unsafe result url',
+      response: {
+        ok: true,
+        status: 200,
+        headers: {},
+        body: JSON.stringify({
+          results: [{ url: 'file:///etc/passwd', raw_content: 'bad' }],
+          failed_results: [],
+        }),
+      },
+      expected: /格式异常/,
+    },
+    {
+      name: 'empty content',
+      response: {
+        ok: true,
+        status: 200,
+        headers: {},
+        body: JSON.stringify({
+          results: [{ url: 'https://example.test/article', raw_content: '   ' }],
+          failed_results: [],
+        }),
+      },
+      expected: /没有可读取/,
+    },
+  ];
+  for (const testCase of cases) {
+    await t.test(testCase.name, async () => {
+      const harness = createHarness({ networkResult: () => testCase.response });
+      const result = await harness.fetchPage();
+      assert.equal(result.ok, false);
+      assert.match(result.message, testCase.expected);
+      assert.doesNotMatch(result.message, /sensitive-name|upstream/);
+    });
+  }
+});
+
+test('fetch_page hides unexpected runtime errors behind an actionable message', async () => {
+  const harness = createHarness({ networkResult: () => null });
+  const result = await harness.fetchPage();
+
+  assert.equal(result.type, 'tool-result');
+  assert.equal(result.callId, 'call-1');
+  assert.equal(result.ok, false);
+  assert.equal(result.message, '网页正文读取失败，请稍后重试');
+  assert.doesNotMatch(result.message, /search|搜索|TypeError|Cannot read/i);
+});
+
 test('missing provider uses Cindy AI by default and does not touch BYO network', async () => {
   const harness = createHarness();
   const result = await harness.search();
@@ -316,6 +624,11 @@ test('explicit provider wins over settings and provider failures never fall back
   assert.equal(failedResult.message, 'Cindy AI quota exhausted');
 });
 
+test('settings explains that Tavily independently enables page reading', () => {
+  assert.match(settingsHtml, /Tavily Key 还会独立启用网页正文读取/);
+  assert.match(settingsHtml, /即使保持 Cindy AI 搜索也可单独使用正文读取/);
+});
+
 test('settings controls stay disabled until initial preferences load', async () => {
   const pendingKv = deferred();
   const harness = createSettingsHarness({

README.md

@@ -40,7 +40,7 @@ This is the source for every official plugin (Ghost) in the
 | <img src="./cindy-gitlab/assets/icon.png" width="22" alt=""> | GitLab | [`cindy-gitlab`](./cindy-gitlab) | GitLab (gitlab.com and self-hosted) issues / MRs / repository operations |
 | <img src="./cindy-mermaid/assets/icon.jpg" width="22" alt=""> | Mermaid | [`cindy-mermaid`](./cindy-mermaid) | Mermaid diagram source normalization and common syntax fixes |
 | <img src="./cindy-notion/assets/icon.png" width="22" alt=""> | Notion | [`cindy-notion`](./cindy-notion) | Read/write Notion pages, databases, and knowledge bases |
-| <img src="./cindy-web-search/assets/icon.png" width="22" alt=""> | Web Search | [`cindy-web-search`](./cindy-web-search) | Public web search (Cindy AI by default; optional user-provided Brave / Tavily key) |
+| <img src="./cindy-web-search/assets/icon.png" width="22" alt=""> | Web Search | [`cindy-web-search`](./cindy-web-search) | Public web search (Cindy AI by default; optional Brave / Tavily key), plus reading any single public HTTP(S) page with a Tavily key |
 | <img src="./world-bank-open-data/assets/icon.png" width="22" alt=""> | World Bank Open Data | [`world-bank-open-data`](./world-bank-open-data) | Public country, economic, social, and development indicators with no API key; staged rollout |
 | <img src="./google-gmail/assets/icon.png" width="22" alt=""> | Gmail | [`google-gmail`](./google-gmail) | Search, read, and organize Gmail, create drafts, and send messages; host-managed OAuth |
 | <img src="./google-drive/assets/icon.png" width="22" alt=""> | Google Drive | [`google-drive`](./google-drive) | Search, read, download, upload, move, and delete Drive files |

README.zh-CN.md

@@ -37,7 +37,7 @@
 | <img src="./cindy-gitlab/assets/icon.png" width="22" alt=""> | GitLab | [`cindy-gitlab`](./cindy-gitlab) | GitLab（gitlab.com 及自建实例）issue / MR / 仓库操作 |
 | <img src="./cindy-mermaid/assets/icon.jpg" width="22" alt=""> | Mermaid | [`cindy-mermaid`](./cindy-mermaid) | Mermaid 图表源码规范化与常见语法修复 |
 | <img src="./cindy-notion/assets/icon.png" width="22" alt=""> | Notion | [`cindy-notion`](./cindy-notion) | Notion 页面、数据库与知识库读写 |
-| <img src="./cindy-web-search/assets/icon.png" width="22" alt=""> | Web Search | [`cindy-web-search`](./cindy-web-search) | 公网搜索（默认 Cindy AI，可选用户自备 Brave / Tavily Key） |
+| <img src="./cindy-web-search/assets/icon.png" width="22" alt=""> | Web Search | [`cindy-web-search`](./cindy-web-search) | 公网搜索（默认 Cindy AI，可选 Brave / Tavily Key），以及使用 Tavily Key 读取任意单个公开 HTTP(S) 网页正文 |
 | <img src="./world-bank-open-data/assets/icon.png" width="22" alt=""> | 世界银行公开数据 | [`world-bank-open-data`](./world-bank-open-data) | 无需 API Key，查询全球国家、经济、社会与发展指标；定向灰度 |
 | <img src="./google-gmail/assets/icon.png" width="22" alt=""> | Gmail | [`google-gmail`](./google-gmail) | 搜索、阅读、整理 Gmail 邮件，生成草稿或发送邮件；授权由宿主托管 |
 | <img src="./google-drive/assets/icon.png" width="22" alt=""> | Google Drive | [`google-drive`](./google-drive) | 搜索、读取、下载、上传、移动和删除云端文件 |