visitFiles; build sidebar

mbostock · mbostock · commit f7e0ce6c1fef · 2023-10-08T20:29:04.000-07:00
diff --git a/public/style.css b/public/style.css
@@ -103,7 +103,7 @@ main {
   #observablehq-sidebar {
     display: initial;
   }
-  #observablehq-center {
+  #observablehq-center.observablehq--sidebar {
     padding-left: calc(240px + 2rem);
   }
 }
diff --git a/src/build.ts b/src/build.ts
@@ -1,170 +1,105 @@
-import {copyFile, mkdir, readFile, readdir, stat, writeFile} from "node:fs/promises";
-import {basename, dirname, join, normalize} from "node:path";
+import {access, constants, copyFile, mkdir, readFile, writeFile} from "node:fs/promises";
+import {basename, dirname, join, normalize, relative} from "node:path";
+import {cwd} from "node:process";
 import {fileURLToPath} from "node:url";
 import {parseArgs} from "node:util";
-import {isNodeError} from "./error.js";
+import {visitFiles, visitMarkdownFiles} from "./files.js";
 import {renderServerless} from "./render.js";
+import {readPages} from "./navigation.js";
 
 const EXTRA_FILES = new Map([["node_modules/@observablehq/runtime/dist/runtime.js", "_observablehq/runtime.js"]]);
 
 async function build(context: CommandContext) {
-  const {root = "./docs", output = "dist", files} = context;
-
-  const sourceRootDirectory = normalize(root);
-  const outputDirectory = normalize(output);
-
-  if (files.length === 0) {
-    files.push(sourceRootDirectory);
-  }
-
-  const sources: {
-    outputPath: string;
-    sourcePath: string;
-    content: string;
-  }[] = [];
+  const {sourceRoot, outputRoot} = context;
 
   // Make sure all files are readable before starting to write output files.
-  await visitFiles(files, outputDirectory, sourceRootDirectory, async (sourcePath, outputPath) => {
-    if (!sourcePath.endsWith(".md")) return;
-    outputPath = outputPath.replace(/\.md$/, ".html");
-    try {
-      const content = await readFile(sourcePath, "utf-8");
-      sources.push({sourcePath, outputPath, content});
-    } catch (error) {
-      throw new Error(`Unable to read ${sourcePath}: ${isNodeError(error) ? error.message : "unknown error"}`);
-    }
-  });
+  for await (const sourceFile of visitMarkdownFiles(sourceRoot)) {
+    await access(join(sourceRoot, sourceFile), constants.R_OK);
+  }
 
   // Render .md files, building a list of file attachments as we go.
-  const fileAttachments: {name: string; mimeType: string}[] = [];
-  for (const {content, outputPath, sourcePath} of sources) {
+  const pages = await readPages(sourceRoot);
+  const files: string[] = [];
+  for await (const sourceFile of visitMarkdownFiles(sourceRoot)) {
+    const sourcePath = join(sourceRoot, sourceFile);
+    const outputPath = join(outputRoot, join(dirname(sourceFile), basename(sourceFile, ".md") + ".html"));
     console.log("render", sourcePath, "→", outputPath);
-    const render = renderServerless(content);
-    fileAttachments.push(...render.files.map((f) => ({...f, sourcePath})));
-    const outputDirectory = outputPath.lastIndexOf("/") > 0 ? outputPath.slice(0, outputPath.lastIndexOf("/")) : null;
-    if (outputDirectory) {
-      try {
-        await mkdir(outputDirectory, {recursive: true});
-      } catch (error) {
-        throw new Error(
-          `Unable to create output directory ${outputDirectory}: ${
-            isNodeError(error) ? error.message : "unknown error"
-          }`
-        );
-      }
-    }
+    const path = `/${join(dirname(sourceFile), basename(sourceFile, ".md"))}`;
+    const render = renderServerless(await readFile(sourcePath, "utf-8"), {path, pages});
+    files.push(...render.files.map((f) => join(sourceFile, "..", f.name)));
+    await prepareOutput(outputPath);
     await writeFile(outputPath, render.html);
   }
 
-  // Copy over the ../public directory.
-  const publicPath = join(dirname(fileURLToPath(import.meta.url)), "..", "public");
-  await visitFiles(publicPath, outputDirectory + "/_observablehq", publicPath, (sourcePath, outputPath) => {
+  // Copy over the public directory.
+  const publicRoot = join(dirname(relative(cwd(), fileURLToPath(import.meta.url))), "..", "public");
+  for await (const publicFile of visitFiles(publicRoot)) {
+    const sourcePath = join(publicRoot, publicFile);
+    const outputPath = join(outputRoot, "_observablehq", publicFile);
     console.log("copy", sourcePath, "→", outputPath);
-    return copyFile(sourcePath, outputPath);
-  });
+    await prepareOutput(outputPath);
+    await copyFile(sourcePath, outputPath);
+  }
 
-  // Copy over the referenced files
-  // TODO: This needs more work and consideration for nested directories.
-  await visitFiles(files, outputDirectory + "/_file", sourceRootDirectory, async (sourcePath, outputPath) => {
-    const sourceName = basename(sourcePath);
-    if (fileAttachments.some((f) => f.name === sourceName)) {
-      console.log("copy", sourcePath, "→", outputPath);
-      return copyFile(sourcePath, outputPath);
-    }
-  });
+  // Copy over the referenced files.
+  for (const file of files) {
+    const sourcePath = join(sourceRoot, file);
+    const outputPath = join(outputRoot, "_file", file);
+    console.log("copy", sourcePath, "→", outputPath);
+    await prepareOutput(outputPath);
+    await copyFile(sourcePath, outputPath);
+  }
 
   // Copy over required distribution files from node_modules.
   // TODO: Note that this requires that the build command be run relative to the node_modules directory.
-  for (const [sourcePath, targetPath] of EXTRA_FILES) {
-    const outputPath = join(outputDirectory, targetPath);
+  for (const [sourcePath, targetFile] of EXTRA_FILES) {
+    const outputPath = join(outputRoot, targetFile);
     console.log("copy", sourcePath, "→", outputPath);
+    await prepareOutput(outputPath);
     await copyFile(sourcePath, outputPath);
   }
 }
 
-async function visitFiles(
-  source: string | string[],
-  output: string,
-  root: string,
-  visitor: (sourcePath: string, outputPath: string) => Promise<void>
-) {
-  const sourceRootDirectory = normalize(root);
-  const outputDirectory = normalize(output);
-
-  const visited = new Set<number>();
-  const files: string[] = Array.isArray(source) ? source.map((file) => normalize(file)) : [normalize(source)];
-
-  for (const file of files) {
-    const sourcePath = normalize(file);
-    const status = await stat(sourcePath);
-    if (status.isDirectory()) {
-      if (visited.has(status.ino)) throw new Error("Circular directory structure with " + sourcePath);
-      visited.add(status.ino);
-      for (const entry of await readdir(sourcePath)) {
-        files.push(join(sourcePath, entry));
-      }
-      continue;
-    }
-
-    const subPath = sourcePath.startsWith(sourceRootDirectory + "/")
-      ? sourcePath.slice(sourceRootDirectory.length + 1)
-      : sourcePath;
-    const outputPath = join(outputDirectory, subPath);
-
-    const dest = outputPath.lastIndexOf("/") > 0 ? outputPath.slice(0, outputPath.lastIndexOf("/")) : null;
-    if (dest) {
-      try {
-        await mkdir(dest, {recursive: true});
-      } catch (error) {
-        throw new Error(
-          `Unable to create output directory ${dest}: ${isNodeError(error) ? error.message : "unknown error"}`
-        );
-      }
-    }
-
-    await visitor(sourcePath, outputPath);
-  }
+async function prepareOutput(outputPath: string): Promise<void> {
+  const outputDir = dirname(outputPath);
+  if (outputDir === ".") return;
+  await mkdir(outputDir, {recursive: true});
 }
 
-// TODO We also need to copy over any referenced file attachments; these live in
-// ./dist/_file (currently; perhaps they should be somewhere else)?
-
-const USAGE = `Usage: observable build [--root dir] [--output dir] [files...]`;
+const USAGE = `Usage: observable build [--root dir] [--output dir]`;
 
 interface CommandContext {
-  root?: string;
-  output?: string;
-  files: string[];
+  sourceRoot: string;
+  outputRoot: string;
 }
 
 function makeCommandContext(): CommandContext {
-  const {values, positionals} = parseArgs({
-    allowPositionals: true,
+  const {values} = parseArgs({
     options: {
       root: {
         type: "string",
-        short: "r"
+        short: "r",
+        default: "docs"
       },
       output: {
         type: "string",
-        short: "o"
+        short: "o",
+        default: "dist"
       }
     }
   });
-
+  if (!values.root || !values.output) {
+    console.error(USAGE);
+    process.exit(1);
+  }
   return {
-    root: values.root,
-    output: values.output,
-    files: positionals
+    sourceRoot: normalize(values.root),
+    outputRoot: normalize(values.output)
   };
 }
 
 await (async function () {
   const context = makeCommandContext();
-  if (!context.files.length && !context.root) {
-    console.error(USAGE);
-    process.exit(1);
-  }
   await build(context);
   process.exit(0);
 })();
diff --git a/src/files.ts b/src/files.ts
@@ -0,0 +1,26 @@
+import {readdir, stat} from "node:fs/promises";
+import {extname, join, normalize, relative} from "node:path";
+
+export async function* visitMarkdownFiles(root: string): AsyncGenerator<string> {
+  for await (const file of visitFiles(root)) {
+    if (extname(file) !== ".md") continue;
+    yield file;
+  }
+}
+
+export async function* visitFiles(root: string): AsyncGenerator<string> {
+  const visited = new Set<number>();
+  const queue: string[] = [(root = normalize(root))];
+  for (const path of queue) {
+    const status = await stat(path);
+    if (status.isDirectory()) {
+      if (visited.has(status.ino)) throw new Error(`Circular directory: ${path}`);
+      visited.add(status.ino);
+      for (const entry of await readdir(path)) {
+        queue.push(join(path, entry));
+      }
+    } else {
+      yield relative(root, path);
+    }
+  }
+}
diff --git a/src/navigation.ts b/src/navigation.ts
@@ -0,0 +1,26 @@
+import {readFile} from "node:fs/promises";
+import {basename, dirname, extname, join} from "node:path";
+import {isNodeError} from "./error.js";
+import {visitFiles} from "./files.js";
+import {parseMarkdown, type ParseResult} from "./markdown.js";
+import {type RenderOptions} from "./render.js";
+
+// TODO Global configuration file? Watcher?
+export async function readPages(root: string): Promise<NonNullable<RenderOptions["pages"]>> {
+  const pages: RenderOptions["pages"] = [];
+  for await (const file of visitFiles(root)) {
+    if (extname(file) !== ".md") continue;
+    let parsed: ParseResult;
+    try {
+      parsed = parseMarkdown(await readFile(join(root, file), "utf-8"));
+    } catch (error) {
+      if (!isNodeError(error) || error.code !== "ENOENT") throw error; // internal error
+      continue;
+    }
+    const name = basename(file, ".md");
+    const page = {path: `/${join(dirname(file), name)}`, name: parsed.title ?? "Untitled"};
+    if (name === "index") pages.unshift(page);
+    else pages.push(page);
+  }
+  return pages;
+}
diff --git a/src/preview.ts b/src/preview.ts
@@ -1,5 +1,5 @@
 import {watch, type FSWatcher} from "node:fs";
-import {readFile, readdir, stat} from "node:fs/promises";
+import {readFile, stat} from "node:fs/promises";
 import type {IncomingMessage, RequestListener} from "node:http";
 import {createServer} from "node:http";
 import {basename, dirname, extname, join, normalize} from "node:path";
@@ -9,8 +9,8 @@ import send from "send";
 import {WebSocketServer, type WebSocket} from "ws";
 import {HttpError, isHttpError, isNodeError} from "./error.js";
 import {computeHash} from "./hash.js";
-import {type ParseResult, parseMarkdown} from "./markdown.js";
-import {type RenderOptions, renderPreview} from "./render.js";
+import {readPages} from "./navigation.js";
+import {renderPreview} from "./render.js";
 
 const DEFAULT_ROOT = "docs";
 
@@ -76,7 +76,7 @@ class Server {
         // Otherwise, serve the corresponding Markdown file, if it exists.
         // Anything else should 404; static files should be matched above.
         try {
-          const pages = await this._readPages(); // TODO cache
+          const pages = await readPages(this.root); // TODO cache? watcher?
           res.end(renderPreview(await readFile(path + ".md", "utf-8"), {path: pathname, pages}).html);
         } catch (error) {
           if (!isNodeError(error) || error.code !== "ENOENT") throw error; // internal error
@@ -91,24 +91,6 @@ class Server {
     }
   };
 
-  async _readPages() {
-    const pages: RenderOptions["pages"] = [];
-    for (const file of await readdir(this.root)) {
-      if (extname(file) !== ".md") continue;
-      let parsed: ParseResult;
-      try {
-        parsed = parseMarkdown(await readFile(join(this.root, file), "utf-8"));
-      } catch (error) {
-        if (!isNodeError(error) || error.code !== "ENOENT") throw error; // internal error
-        continue;
-      }
-      const page = {path: `/${basename(file, ".md")}`, name: parsed.title ?? "Untitled"};
-      if (page.path === "/index") pages.unshift(page);
-      else pages.push(page);
-    }
-    return pages;
-  }
-
   _handleConnection = (socket: WebSocket, req: IncomingMessage) => {
     if (req.url === "/_observablehq") {
       handleWatch(socket, this.root);
diff --git a/src/render.ts b/src/render.ts
@@ -1,6 +1,6 @@
+import {computeHash} from "./hash.js";
 import type {ParseResult} from "./markdown.js";
 import {parseMarkdown} from "./markdown.js";
-import {computeHash} from "./hash.js";
 
 export interface Render {
   html: string;
@@ -27,6 +27,7 @@ type RenderInternalOptions =
   | {preview: true; hash: string}; // preview
 
 function render(parseResult: ParseResult, {path, pages, preview, hash}: RenderOptions & RenderInternalOptions): string {
+  const showSidebar = pages && pages.length > 1;
   return `<!DOCTYPE html>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
@@ -46,7 +47,7 @@ ${JSON.stringify(parseResult.data)}
       : ""
   }
 ${
-  pages
+  showSidebar
     ? `<nav id="observablehq-sidebar">
   <ol>${pages
     ?.map(
@@ -60,7 +61,7 @@ ${
 </nav>
 `
     : ""
-}<div id="observablehq-center">
+}<div id="observablehq-center"${showSidebar ? ` class="observablehq--sidebar"` : ""}>
 <main>
 ${parseResult.html}</main>
 </div>

Original file line number	Diff line number	Diff line change
`@@ -103,7 +103,7 @@ main {`
`103`	`103`	`#observablehq-sidebar {`
`104`	`104`	`display: initial;`
`105`	`105`	`}`
`106`		`- #observablehq-center {`
	`106`	`+ #observablehq-center.observablehq--sidebar {`
`107`	`107`	`padding-left: calc(240px + 2rem);`
`108`	`108`	`}`
`109`	`109`	`}`