diff --git a/.gitignore b/.gitignore index cd8400f..49276f1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ # Build output dist/ +.astro/ # Lock files (using npm) package-lock.json diff --git a/packages/scribe/README.md b/packages/scribe/README.md new file mode 100644 index 0000000..0aea54d --- /dev/null +++ b/packages/scribe/README.md @@ -0,0 +1,57 @@ +# @sailkit/scribe + +Test code fences from markdown documentation. + +**READ-ONLY FILESYSTEM SAFE** - No temp files, no disk writes. Everything stays in memory. + +## Usage + +```bash +scribe # Test a file or directory +scribe docs/ # Scan all .md/.mdx files +scribe README.md # Test a single file +``` + +All JavaScript/TypeScript code blocks are tested by default. Use `nocheck` to skip: + +~~~markdown +```typescript nocheck +// This block won't be tested +``` +~~~ + +## Writing Testable Examples + +Code blocks should be self-contained and use assertions. Node's `assert` module is globally available: + +```typescript +const items = ['a', 'b', 'c'] +assert.strictEqual(items[0], 'a') +assert.deepStrictEqual(items, ['a', 'b', 'c']) +``` + +No import needed - `assert` is injected by scribe. Code remains copy-pasteable since `assert` is a standard Node module. + +## CLI Options + +- `-i, --runInBand` - Run tests sequentially (no parallelism) +- `-j, --parallel ` - Set number of parallel workers (default: CPU count) + +## API + +```typescript +import { parseMarkdown, filterTestableBlocks, runBlocks } from '@sailkit/scribe' + +const blocks = parseMarkdown(content, 'file.md') +const testable = filterTestableBlocks(blocks) +const results = await runBlocks(testable) +``` + +## Future + +Scribe will evolve beyond static analysis to enable interactive documentation: + +- **In-browser execution** - Run code blocks directly on the page +- **Live verification** - See pass/fail results as you read +- **Monaco Editor integration** - Edit and re-run examples inline +- **Playground mode** - Experiment with code without leaving the docs diff --git a/packages/scribe/package.json b/packages/scribe/package.json new file mode 100644 index 0000000..865801e --- /dev/null +++ b/packages/scribe/package.json @@ -0,0 +1,31 @@ +{ + "name": "@sailkit/scribe", + "version": "0.0.1", + "description": "Extract and test code fences from markdown documentation", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "bin": { + "scribe": "dist/cli.js" + }, + "scripts": { + "build": "tsc", + "test": "vitest run" + }, + "keywords": [ + "documentation", + "testing", + "markdown", + "code-fence" + ], + "license": "MIT", + "devDependencies": { + "typescript": "^5.0.0", + "vitest": "^4.0.15" + }, + "dependencies": { + "commander": "^14.0.2", + "esbuild": "^0.27.1", + "log-update": "^7.0.2" + } +} diff --git a/packages/scribe/src/cli.test.ts b/packages/scribe/src/cli.test.ts new file mode 100644 index 0000000..8167f03 --- /dev/null +++ b/packages/scribe/src/cli.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, beforeEach } from 'vitest' +import { run, type IO, type RunOptions } from './cli.js' +import { writeFile, mkdir, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' + +// Mock IO that captures output +function createMockIO(): IO & { stdout: string; stderr: string } { + const io = { + stdout: '', + stderr: '', + write(text: string) { + io.stdout += text + }, + writeError(text: string) { + io.stderr += text + } + } + return io +} + +describe('scribe CLI', () => { + let testDir: string + + beforeEach(async () => { + testDir = join(tmpdir(), `scribe-test-${Date.now()}`) + await mkdir(testDir, { recursive: true }) + }) + + it('reports error for non-existent path', async () => { + const io = createMockIO() + const result = await run({ targetPath: '/nonexistent/path', concurrency: 1 }, io) + + expect(result.failed).toBe(1) + expect(io.stderr).toContain('Error: Path not found') + }) + + it('reports no testable blocks for markdown without code fences', async () => { + const mdPath = join(testDir, 'empty.md') + await writeFile(mdPath, '# Hello\n\nNo code here.') + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 1 }, io) + + expect(result.passed).toBe(0) + expect(result.failed).toBe(0) + expect(io.stdout).toContain('No testable code blocks found') + }) + + it('runs passing JavaScript code blocks', async () => { + const mdPath = join(testDir, 'passing.md') + await writeFile(mdPath, `# Test + +\`\`\`javascript +console.log("hello") +\`\`\` +`) + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 1 }, io) + + expect(result.passed).toBe(1) + expect(result.failed).toBe(0) + expect(io.stdout).toContain('Results: 1/1 passed') + }) + + it('runs passing TypeScript code blocks', async () => { + const mdPath = join(testDir, 'typescript.md') + await writeFile(mdPath, `# TypeScript Test + +\`\`\`typescript +const x: number = 42 +console.log(x) +\`\`\` +`) + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 1 }, io) + + expect(result.passed).toBe(1) + expect(result.failed).toBe(0) + }) + + it('reports failing code blocks', async () => { + const mdPath = join(testDir, 'failing.md') + await writeFile(mdPath, `# Failing Test + +\`\`\`javascript +throw new Error("oops") +\`\`\` +`) + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 1 }, io) + + expect(result.passed).toBe(0) + expect(result.failed).toBe(1) + expect(io.stdout).toContain('Results: 0/1 passed') + }) + + it('scans directories for markdown files', async () => { + await writeFile(join(testDir, 'a.md'), `\`\`\`js +console.log(1) +\`\`\``) + await writeFile(join(testDir, 'b.md'), `\`\`\`js +console.log(2) +\`\`\``) + + const io = createMockIO() + const result = await run({ targetPath: testDir, concurrency: 1 }, io) + + expect(result.passed).toBe(2) + expect(result.failed).toBe(0) + expect(io.stdout).toContain('Found 2 markdown file(s)') + }) + + it('ignores code blocks with nocheck marker', async () => { + const mdPath = join(testDir, 'nocheck.md') + await writeFile(mdPath, `# Test + +\`\`\`javascript nocheck +// This should be ignored +throw new Error("not run") +\`\`\` + +\`\`\`javascript +console.log("this runs") +\`\`\` +`) + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 1 }, io) + + expect(result.passed).toBe(1) + expect(result.failed).toBe(0) + }) + + it('runs multiple blocks in parallel when concurrency > 1', async () => { + const mdPath = join(testDir, 'parallel.md') + await writeFile(mdPath, ` +\`\`\`js +console.log(1) +\`\`\` + +\`\`\`js +console.log(2) +\`\`\` + +\`\`\`js +console.log(3) +\`\`\` +`) + + const io = createMockIO() + const result = await run({ targetPath: mdPath, concurrency: 3 }, io) + + expect(result.passed).toBe(3) + expect(result.failed).toBe(0) + }) +}) diff --git a/packages/scribe/src/cli.ts b/packages/scribe/src/cli.ts new file mode 100644 index 0000000..dafdfd3 --- /dev/null +++ b/packages/scribe/src/cli.ts @@ -0,0 +1,262 @@ +#!/usr/bin/env node +/** + * IMPORTANT: THIS MODULE MUST BE READ-ONLY FILESYSTEM SAFE + * No temp files, no disk writes - everything stays in memory. + * + * Scribe CLI - test code fences from markdown files + */ + +import { readFile, readdir, stat } from 'node:fs/promises' +import { join, extname } from 'node:path' +import { cpus } from 'node:os' +import { Command } from 'commander' +import logUpdate from 'log-update' +import { parseMarkdown, filterTestableBlocks, type CodeBlock } from './parser.js' +import { runBlock, type RunResult } from './runner.js' + +// IO interface for testability +export interface IO { + write(text: string): void + writeError(text: string): void +} + +// Real IO implementation +const realIO: IO = { + write: (text) => process.stdout.write(text), + writeError: (text) => process.stderr.write(text) +} + +// Detect interactive TTY vs CI/piped output +const isTTY = process.stdout.isTTY ?? false +const isCI = process.env.CI === 'true' || process.env.GITHUB_ACTIONS === 'true' +const useInteractiveOutput = isTTY && !isCI + +// Spinner frames +const spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] +let spinnerIndex = 0 + +interface RunningTest { + label: string + index: number +} + +export interface RunOptions { + targetPath: string + concurrency: number +} + +async function findMarkdownFiles(dir: string): Promise { + const files: string[] = [] + + async function walk(currentDir: string) { + const entries = await readdir(currentDir) + for (const entry of entries) { + const fullPath = join(currentDir, entry) + const info = await stat(fullPath) + + if (info.isDirectory() && !entry.startsWith('.') && entry !== 'node_modules') { + await walk(fullPath) + } else if (info.isFile() && ['.md', '.mdx'].includes(extname(entry))) { + files.push(fullPath) + } + } + } + + await walk(dir) + return files +} + +function cleanErrorMessage(error: string): string { + return error + .replace(/file:\/\/[^\s]+\[eval\d*\]:\d+/g, '') + .replace(/\[stdin\]:\d+/g, '') + .split('\n') + .filter(line => !line.includes('node:internal') && !line.includes('file://')) + .map(line => line.trim()) + .filter(Boolean) + .slice(0, 1) + .join('') +} + +function renderRunningTests(running: RunningTest[]): string { + if (running.length === 0) return '' + + const spinner = spinnerFrames[spinnerIndex % spinnerFrames.length] + const lines = running.map(t => ` \x1b[33m${spinner}\x1b[0m ${t.label}`) + return lines.join('\n') +} + +export async function run(options: RunOptions, io: IO = realIO): Promise<{ passed: number; failed: number }> { + const { targetPath, concurrency } = options + + // Determine if target is file or directory + const targetStat = await stat(targetPath).catch(() => null) + if (!targetStat) { + io.writeError(`Error: Path not found: ${targetPath}\n`) + return { passed: 0, failed: 1 } + } + + let mdFiles: string[] + let baseDir: string + + if (targetStat.isFile()) { + mdFiles = [targetPath] + baseDir = targetPath.includes('/') ? targetPath.substring(0, targetPath.lastIndexOf('/')) : '.' + } else { + io.write('Scanning for markdown files...\n') + mdFiles = await findMarkdownFiles(targetPath) + baseDir = targetPath + } + + io.write(`Found ${mdFiles.length} markdown file(s) (${concurrency} workers)\n\n`) + + // Collect all blocks + const allBlocks: { block: CodeBlock; shortFile: string }[] = [] + + for (const file of mdFiles) { + const content = await readFile(file, 'utf-8') + const blocks = filterTestableBlocks(parseMarkdown(content, file)) + // Get relative path for display + let shortFile = file + if (baseDir !== '.' && file.startsWith(baseDir)) { + shortFile = file.slice(baseDir.length).replace(/^\//, '') + } + + for (const block of blocks) { + allBlocks.push({ block, shortFile }) + } + } + + if (allBlocks.length === 0) { + io.write('No testable code blocks found.\n') + return { passed: 0, failed: 0 } + } + + // Track state + const runningTests: Map = new Map() + let passed = 0 + let failed = 0 + + // Spinner animation interval (only updates the bottom running section) + let animationInterval: ReturnType | undefined + if (useInteractiveOutput) { + animationInterval = setInterval(() => { + spinnerIndex++ + const running = Array.from(runningTests.values()) + if (running.length > 0) { + logUpdate(renderRunningTests(running)) + } + }, 80) + } + + // Helper to print a completed test (goes into scrollback) + function printCompleted(label: string, success: boolean, error?: string) { + if (useInteractiveOutput) { + logUpdate.clear() + } + + if (success) { + io.write(`\x1b[32m✓\x1b[0m ${label}\n`) + } else { + io.write(`\x1b[31m✗\x1b[0m ${label}\n`) + if (error) { + io.write(` \x1b[90m${error}\x1b[0m\n`) + } + } + + if (useInteractiveOutput) { + const running = Array.from(runningTests.values()) + if (running.length > 0) { + logUpdate(renderRunningTests(running)) + } + } + } + + // Run with concurrency + const results: RunResult[] = new Array(allBlocks.length) + let nextIndex = 0 + + async function worker() { + while (nextIndex < allBlocks.length) { + const index = nextIndex++ + const { shortFile, block } = allBlocks[index] + const label = `${shortFile}:${block.line}` + + runningTests.set(index, { label, index }) + + if (useInteractiveOutput) { + logUpdate(renderRunningTests(Array.from(runningTests.values()))) + } + + const result = await runBlock(block) + results[index] = result + + runningTests.delete(index) + + const cleanError = result.error ? cleanErrorMessage(result.error) : undefined + + if (result.success) { + passed++ + printCompleted(label, true) + } else { + failed++ + printCompleted(label, false, cleanError) + } + } + } + + const workers = Array.from( + { length: Math.min(concurrency, allBlocks.length) }, + () => worker() + ) + await Promise.all(workers) + + if (animationInterval) { + clearInterval(animationInterval) + } + + if (useInteractiveOutput) { + logUpdate.clear() + } + + io.write('\n') + const color = failed > 0 ? '\x1b[31m' : '\x1b[32m' + io.write(`${color}Results: ${passed}/${allBlocks.length} passed\x1b[0m\n`) + + return { passed, failed } +} + +// CLI entry point +const program = new Command() + .name('scribe') + .description('Test code fences from markdown files') + .argument('', 'File or directory to test') + .option('-i, --runInBand', 'Run tests sequentially (no parallelism)') + .option('-j, --parallel ', 'Set number of parallel workers', String(cpus().length)) + +async function main() { + program.parse() + + const targetPath = program.args[0] + const opts = program.opts() + + let concurrency = parseInt(opts.parallel, 10) || cpus().length + if (opts.runInBand) { + concurrency = 1 + } + + const { failed } = await run({ targetPath, concurrency }) + + if (failed > 0) { + process.exit(1) + } +} + +// Only run when executed directly, not when imported for testing +const isMainModule = import.meta.url === `file://${process.argv[1]}` +if (isMainModule) { + main().catch((err) => { + console.error('Scribe error:', err.message) + process.exit(1) + }) +} diff --git a/packages/scribe/src/index.ts b/packages/scribe/src/index.ts new file mode 100644 index 0000000..fbdf5d8 --- /dev/null +++ b/packages/scribe/src/index.ts @@ -0,0 +1,13 @@ +/** + * IMPORTANT: THIS PACKAGE MUST BE READ-ONLY FILESYSTEM SAFE + * No temp files, no disk writes - everything stays in memory. + * + * @sailkit/scribe + * Test code fences from markdown documentation + */ + +export { parseMarkdown, filterTestableBlocks } from './parser.js' +export type { CodeBlock } from './parser.js' + +export { runBlock, runBlocks } from './runner.js' +export type { RunResult } from './runner.js' diff --git a/packages/scribe/src/parser.ts b/packages/scribe/src/parser.ts new file mode 100644 index 0000000..cd95b18 --- /dev/null +++ b/packages/scribe/src/parser.ts @@ -0,0 +1,51 @@ +/** + * IMPORTANT: THIS MODULE MUST BE READ-ONLY FILESYSTEM SAFE + * No temp files, no disk writes - everything stays in memory. + * + * Code fence parser - extracts fenced code blocks from markdown + */ + +export interface CodeBlock { + language: string + code: string + file: string + line: number + meta?: string +} + +// Matches code fences with optional info string (e.g., ```javascript scribe) +// Captures: 1=language, 2=rest of info line (may contain "scribe" marker), 3=code +const CODE_FENCE_REGEX = /^```(\w*)([^\n]*)\n([\s\S]*?)^```/gm + +export function parseMarkdown(content: string, filePath: string): CodeBlock[] { + const blocks: CodeBlock[] = [] + let match: RegExpExecArray | null + + while ((match = CODE_FENCE_REGEX.exec(content)) !== null) { + const language = match[1] || 'text' + const infoString = match[2]?.trim() || '' + const code = match[3] + + // Calculate line number + const beforeMatch = content.slice(0, match.index) + const line = beforeMatch.split('\n').length + + blocks.push({ + language, + code, + file: filePath, + line, + meta: infoString + }) + } + + return blocks +} + +export function filterTestableBlocks(blocks: CodeBlock[]): CodeBlock[] { + const testableLanguages = ['typescript', 'ts', 'javascript', 'js'] + return blocks.filter(block => + testableLanguages.includes(block.language.toLowerCase()) && + !block.meta?.includes('nocheck') // opt-out with ```ts nocheck + ) +} diff --git a/packages/scribe/src/runner.ts b/packages/scribe/src/runner.ts new file mode 100644 index 0000000..ae621d5 --- /dev/null +++ b/packages/scribe/src/runner.ts @@ -0,0 +1,95 @@ +/** + * IMPORTANT: THIS MODULE MUST BE READ-ONLY FILESYSTEM SAFE + * No temp files, no disk writes - everything stays in memory. + * + * In-process runner using esbuild build + vm module + * Bundles imports so code blocks can use real packages + */ + +import vm from 'node:vm' +import assert from 'node:assert' +import { build } from 'esbuild' +import { dirname } from 'node:path' +import type { CodeBlock } from './parser.js' + +export interface RunResult { + block: CodeBlock + success: boolean + output: string + error?: string +} + +export async function runBlock(block: CodeBlock): Promise { + const isTypeScript = block.language === 'typescript' || block.language === 'ts' + + try { + // Bundle with esbuild using stdin - no temp files + const result = await build({ + stdin: { + contents: block.code, + loader: isTypeScript ? 'ts' : 'js', + resolveDir: dirname(block.file), // resolve imports relative to markdown file + }, + bundle: true, + write: false, + format: 'cjs', + platform: 'node', + target: 'node18', + }) + + const code = result.outputFiles[0].text + + // Capture console output + let stdout = '' + const mockConsole = { + log: (...args: unknown[]) => { stdout += args.map(String).join(' ') + '\n' }, + error: (...args: unknown[]) => { stdout += args.map(String).join(' ') + '\n' }, + warn: (...args: unknown[]) => { stdout += args.map(String).join(' ') + '\n' }, + info: (...args: unknown[]) => { stdout += args.map(String).join(' ') + '\n' } + } + + // Create sandbox with common globals + // assert is provided globally so code blocks can use it without imports + const sandbox = { + console: mockConsole, + assert, + setTimeout, + setInterval, + clearTimeout, + clearInterval, + Buffer, + process: { env: process.env }, + Error, + TypeError, + ReferenceError, + SyntaxError + } + + vm.createContext(sandbox) + + // Execute with timeout + const script = new vm.Script(code, { filename: 'code-block.js' }) + script.runInContext(sandbox, { timeout: 5000 }) + + return { + block, + success: true, + output: stdout + } + } catch (err) { + return { + block, + success: false, + output: '', + error: err instanceof Error ? err.message : String(err) + } + } +} + +export async function runBlocks(blocks: CodeBlock[]): Promise { + const results: RunResult[] = [] + for (const block of blocks) { + results.push(await runBlock(block)) + } + return results +} diff --git a/packages/scribe/tsconfig.json b/packages/scribe/tsconfig.json new file mode 100644 index 0000000..f0423d6 --- /dev/null +++ b/packages/scribe/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src"] +} diff --git a/packages/scribe/vitest.config.ts b/packages/scribe/vitest.config.ts new file mode 100644 index 0000000..7aa4024 --- /dev/null +++ b/packages/scribe/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + exclude: ['dist/**', 'node_modules/**'] + } +})