R&D: browserbase for the web resources...

apps/server/package.json

@@ -47,6 +47,7 @@
 	},
 	"devDependencies": {
 		"@types/bun": "latest",
+		"@types/turndown": "^5.0.6",
 		"@typescript/native-preview": "^7.0.0-dev.20260109.1",
 		"prettier": "^3.7.4"
 	},
@@ -55,13 +56,16 @@
 		"@ai-sdk/google": "^3.0.13",
 		"@ai-sdk/openai": "^3.0.18",
 		"@ai-sdk/openai-compatible": "^2.0.18",
+		"@btca/bb": "workspace:*",
 		"@btca/shared": "workspace:*",
 		"ai": "^6.0.49",
 		"better-result": "^2.6.0",
 		"cheerio": "^1.2.0",
 		"hono": "^4.7.11",
 		"just-bash": "^2.7.0",
 		"opencode-ai": "^1.1.36",
+		"turndown": "^7.2.2",
+		"turndown-plugin-gfm": "^1.0.2",
 		"zod": "^3.25.76"
 	}
 }

apps/server/src/config/config.test.ts

@@ -443,14 +443,14 @@ describe('Config', () => {
 			const config = await Config.load();
 
 			// Update the model
-			await config.updateModel('new-provider', 'new-model');
+			await config.updateModel('opencode', 'new-model');
 
-			expect(config.provider).toBe('new-provider');
+			expect(config.provider).toBe('opencode');
 			expect(config.model).toBe('new-model');
 
 			// CRITICAL: Verify project config was updated
 			const savedProjectConfig = JSON.parse(await fs.readFile(projectConfigPath, 'utf-8'));
-			expect(savedProjectConfig.provider).toBe('new-provider');
+			expect(savedProjectConfig.provider).toBe('opencode');
 			expect(savedProjectConfig.model).toBe('new-model');
 			// Global resources should NOT have leaked into project config
 			expect(savedProjectConfig.resources.length).toBe(0);

apps/server/src/config/index.ts

@@ -464,12 +464,8 @@ export namespace Config {
 			return Result.ok(migrated);
 		});
 
-		const saved = result.match({
-			ok: (value) => value,
-			err: (error) => {
-				throw error;
-			}
-		});
+		if (Result.isError(result)) throw result.error;
+		const saved = result.value;
 
 		Metrics.info('config.legacy.migrated', {
 			newPath: newConfigPath,
@@ -498,12 +494,8 @@ export namespace Config {
 			return Result.ok(stored);
 		});
 
-		return result.match({
-			ok: (value) => value,
-			err: (error) => {
-				throw error;
-			}
-		});
+		if (Result.isError(result)) throw result.error;
+		return result.value;
 	};
 
 	const createDefaultConfig = async (configPath: string): Promise<StoredConfig> => {
@@ -540,12 +532,8 @@ export namespace Config {
 			return Result.ok(defaultStored);
 		});
 
-		return result.match({
-			ok: (value) => value,
-			err: (error) => {
-				throw error;
-			}
-		});
+		if (Result.isError(result)) throw result.error;
+		return result.value;
 	};
 
 	const saveConfig = async (configPath: string, stored: StoredConfig): Promise<void> => {
@@ -555,12 +543,7 @@ export namespace Config {
 			`Failed to save config to: "${configPath}"`,
 			'Check that you have write permissions and the disk is not full.'
 		);
-		result.match({
-			ok: () => undefined,
-			err: (error) => {
-				throw error;
-			}
-		});
+		if (Result.isError(result)) throw result.error;
 	};
 
 	/**

apps/server/src/metrics/index.ts

@@ -46,20 +46,17 @@ export namespace Metrics {
 	): Promise<T> => {
 		const start = performance.now();
 		const result = await Result.tryPromise(fn);
-		return result.match({
-			ok: (value) => {
-				info('span.ok', { name, ms: Math.round(performance.now() - start), ...fields });
-				return value;
-			},
-			err: (cause) => {
-				error('span.err', {
-					name,
-					ms: Math.round(performance.now() - start),
-					...fields,
-					error: errorInfo(cause)
-				});
-				throw cause;
-			}
+		if (!Result.isError(result)) {
+			info('span.ok', { name, ms: Math.round(performance.now() - start), ...fields });
+			return result.value;
+		}
+
+		error('span.err', {
+			name,
+			ms: Math.round(performance.now() - start),
+			...fields,
+			error: errorInfo(result.error)
 		});
+		throw result.error;
 	};
 }

apps/server/src/resources/impls/git.ts

@@ -303,12 +303,11 @@ const gitClone = async (args: {
 		return Result.ok(undefined);
 	});
 
-	result.match({
-		ok: () => undefined,
-		err: (error) => {
-			throw error;
-		}
+	const error = result.match({
+		ok: () => null,
+		err: (error) => error
 	});
+	if (error) throw error;
 };
 
 const gitUpdate = async (args: {
@@ -390,12 +389,11 @@ const gitUpdate = async (args: {
 		return Result.ok(undefined);
 	});
 
-	result.match({
-		ok: () => undefined,
-		err: (error) => {
-			throw error;
-		}
+	const error = result.match({
+		ok: () => null,
+		err: (error) => error
 	});
+	if (error) throw error;
 };
 
 /**
@@ -484,12 +482,11 @@ const ensureGitResource = async (config: BtcaGitResourceArgs): Promise<string> =
 						cause
 					})
 			});
-			mkdirResult.match({
-				ok: () => undefined,
-				err: (error) => {
-					throw error;
-				}
+			const mkdirError = mkdirResult.match({
+				ok: () => null,
+				err: (error) => error
 			});
+			if (mkdirError) throw mkdirError;
 
 			await gitClone({
 				repoUrl: config.url,

apps/server/src/resources/impls/website.test.ts

@@ -64,7 +64,8 @@ describe('Website Resource', () => {
 		ttlHours: 24,
 		resourcesDirectoryPath: tempDir,
 		specialAgentInstructions: '',
-		quiet: true
+		quiet: true,
+		renderer: null
 	});
 
 	it('rejects non-HTTPS website URLs', async () => {
@@ -343,4 +344,129 @@ describe('Website Resource', () => {
 
 		expect(calls).toContain('https://bun.com/docs/runtime/binary-data/.md');
 	});
+
+	it('converts HTML to markdown with lists and code blocks', async () => {
+		withMockFetch({
+			'https://docs.example.com/robots.txt': { body: 'User-agent: *\nAllow: /\n' },
+			'https://docs.example.com/sitemap.xml': {
+				headers: { 'content-type': 'application/xml' },
+				body: '<?xml version="1.0"?><urlset></urlset>'
+			},
+			'https://docs.example.com/docs': {
+				headers: { 'content-type': 'text/html' },
+				body: `
+					<html><head><title>Docs</title></head><body>
+						<main>
+							<h1>Docs</h1>
+							<p>Hello</p>
+							<ul><li>one</li><li>two</li></ul>
+							<pre><code>const x = 1;</code></pre>
+						</main>
+					</body></html>
+				`
+			}
+		});
+
+		const resource = await loadWebsiteResource({ ...baseArgs(), maxPages: 1, maxDepth: 0 });
+		const resourcePath = await resource.getAbsoluteDirectoryPath();
+		const content = await Bun.file(path.join(resourcePath, 'pages/docs.md')).text();
+
+		expect(content).toContain('Source: https://docs.example.com/docs');
+		expect(content).toMatch(/-\s+one/);
+		expect(content).toContain('```');
+	});
+
+	it('uses renderer when the page looks like a SPA shell', async () => {
+		const renderCalls: string[] = [];
+		const renderer = {
+			render: async (url: string) => {
+				renderCalls.push(url);
+				return {
+					finalUrl: url,
+					html: '<html><head><title>Rendered</title></head><body><main><h1>Rendered</h1><p>Hello</p></main></body></html>'
+				};
+			},
+			close: async () => undefined
+		};
+
+		withMockFetch({
+			'https://docs.example.com/robots.txt': { body: 'User-agent: *\nAllow: /\n' },
+			'https://docs.example.com/sitemap.xml': {
+				headers: { 'content-type': 'application/xml' },
+				body: '<?xml version="1.0"?><urlset></urlset>'
+			},
+			'https://docs.example.com/docs': {
+				headers: { 'content-type': 'text/html' },
+				body: '<html><head><title>App</title></head><body><div id="root"></div><script src="/app.js"></script><script></script><script></script><script></script><script></script><script></script></body></html>'
+			}
+		});
+
+		const resource = await loadWebsiteResource({
+			...baseArgs(),
+			name: 'spa',
+			maxPages: 1,
+			maxDepth: 0,
+			renderer
+		});
+		const resourcePath = await resource.getAbsoluteDirectoryPath();
+		const content = await Bun.file(path.join(resourcePath, 'pages/docs.md')).text();
+
+		expect(renderCalls).toEqual(['https://docs.example.com/docs']);
+		expect(content).toContain('Hello');
+	});
+
+	it('enforces a render budget per crawl', async () => {
+		const renderCalls: string[] = [];
+		const renderer = {
+			render: async (url: string) => {
+				renderCalls.push(url);
+				return {
+					finalUrl: url,
+					html: `<html><head><title>Rendered</title></head><body><main><p>RENDERED ${url}</p></main></body></html>`
+				};
+			},
+			close: async () => undefined
+		};
+
+		const urls = Array.from({ length: 40 }, (_, i) => `https://docs.example.com/docs/p${i + 1}`);
+		const routes: MockRoutes = {
+			'https://docs.example.com/robots.txt': { body: 'User-agent: *\nAllow: /\n' },
+			'https://docs.example.com/sitemap.xml': {
+				headers: { 'content-type': 'application/xml' },
+				body: `<?xml version="1.0"?><urlset>${urls
+					.map((u) => `<url><loc>${u}</loc></url>`)
+					.join('')}</urlset>`
+			},
+			'https://docs.example.com/docs': {
+				headers: { 'content-type': 'text/html' },
+				body: '<html><head><title>App</title></head><body><div id="root"></div><script></script><script></script><script></script><script></script><script></script><script></script></body></html>'
+			}
+		};
+
+		for (const url of urls) {
+			routes[url] = {
+				headers: { 'content-type': 'text/html' },
+				body: '<html><head><title>App</title></head><body><div id="root"></div><script></script><script></script><script></script><script></script><script></script><script></script></body></html>'
+			};
+		}
+
+		withMockFetch(routes);
+
+		const resource = await loadWebsiteResource({
+			...baseArgs(),
+			name: 'budget',
+			maxPages: 40,
+			maxDepth: 1,
+			renderer
+		});
+		const resourcePath = await resource.getAbsoluteDirectoryPath();
+
+		expect(renderCalls.length).toBe(25);
+
+		const renderedContent = await Bun.file(path.join(resourcePath, 'pages/docs/p1.md')).text();
+		expect(renderedContent).toContain('RENDERED https://docs.example.com/docs/p1');
+
+		const notRenderedContent = await Bun.file(path.join(resourcePath, 'pages/docs/p30.md')).text();
+		expect(notRenderedContent).not.toContain('RENDERED');
+	});
 });