tamnd · tamnd · Aug 6, 2026 · Aug 6, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,19 @@ All notable changes to kage are recorded here. The format follows
 
 ### Fixed
 
+- `--resume` picks an interrupted crawl back up instead of doing nothing ([#36](https://github.com/tamnd/kage/issues/36)).
+  `state.json` persisted only the visited set, and the frontier was rebuilt purely by re-rendering pages and following their links, which resume exists to avoid.
+  So a resumed run found its seed already visited, `enqueuePage` turned it down, nothing was queued, and the run printed `pages 0` and exited successfully with most of the site still missing.
+  Only sites with a `sitemap.xml` appeared to work, because those URLs are seeded independently on every run.
+  The unfinished frontier is now saved alongside the visited set, with each page's depth so `--max-depth` keeps its meaning across a restart, and re-queued at startup.
+  A run now also reports what it is leaving behind: `resume: 412 pages still to do, rerun to continue`.
+- A page that failed is retried by the next run instead of being lost.
+  Failures were never recorded anywhere, so the only way to pick them up was `--refresh`, which re-renders the entire site.
+  This is the "memory of what failed" asked for in [#36](https://github.com/tamnd/kage/issues/36).
+  A page `robots.txt` disallows is not carried over, since a later run would only fetch `robots.txt` and skip it again.
+- `--max-pages` no longer discards the pages it held back.
+  They stay in the frontier, so `kage clone example.com -p 20` to inspect a site and then `kage clone example.com` to finish it now works as a workflow.
+
 - Saved pages keep their `<!DOCTYPE html>` instead of rendering in quirks mode ([#16](https://github.com/tamnd/kage/issues/16)).
   kage serialises a rendered page as the outerHTML of `<html>`, and a doctype is a sibling of `<html>` rather than a child, so it was never in that string and every page kage has ever written came out without one.
   A document with no doctype is quirks mode in every browser: the box model reverts to the pre-CSS2 IE one and `line-height`, table cell inheritance and `vertical-align` all change, so the saved copy laid out differently from the original, and the `<meta charset>` declaration lost its authority, leaving a reader free to fall back to its locale encoding and mojibake every multibyte character.

diff --git a/clone/cloner.go b/clone/cloner.go
@@ -132,7 +132,7 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 		if err := c.front.load(c.statePth); err != nil {
 			c.logf("resume: could not load state: %v", err)
 		} else if n := c.front.visitedCount(); n > 0 {
-			c.logf("resume: %d pages already done", n)
+			c.logf("resume: %s already done", pagesPlural(n))
 		}
 	}
 
@@ -169,8 +169,15 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 		})
 	}
 
-	// Seed.
+	// Seed. A resumed run also re-queues the frontier the previous run saved on
+	// the way out. Without it the seed is already visited, enqueuePage turns it
+	// down, and since the frontier is otherwise rebuilt only by re-rendering
+	// pages and following their links, the run ends having done nothing at all
+	// (issue #36).
 	c.enqueuePage(ctx, c.seed, 0)
+	if n := c.requeueUnfinished(ctx); n > 0 {
+		c.logf("resume: picking up %s", pagesPlural(n))
+	}
 	if c.cfg.FollowSitemap {
 		c.seedSitemaps(ctx)
 	}
@@ -186,6 +193,11 @@ func (c *Cloner) Run(ctx context.Context) (Result, error) {
 	if c.cfg.Persist {
 		if err := c.front.save(c.statePth); err != nil {
 			c.logf("could not save resume state: %v", err)
+		} else if n := c.front.pendingCount(); n > 0 {
+			// Pages left in the frontier: interrupted, over the page budget, or
+			// failed. Saying so is the "memory of what failed" asked for in issue
+			// #36, and it stops a short run looking like a finished one.
+			c.logf("resume: %s still to do, rerun to continue", pagesPlural(n))
 		}
 	}
 
@@ -268,6 +280,7 @@ func (c *Cloner) processPage(ctx context.Context, j pageItem) {
 	key := c.pageKey(j.u)
 	if c.cfg.RespectRobots && !c.robots.Allowed(j.u.Path) {
 		c.stats.skipped.Add(1)
+		c.front.markDone(key)
 		return
 	}
 	if !c.waitForCrawlDelay(ctx) {
@@ -436,6 +449,32 @@ func classifyError(err error) string {
 	return err.Error()
 }
 
+// pagesPlural renders a page count with the right noun, for log lines that are
+// as likely to say 1 as 1,400.
+func pagesPlural(n int) string {
+	if n == 1 {
+		return "1 page"
+	}
+	return fmt.Sprintf("%d pages", n)
+}
+
+// requeueUnfinished puts the previous run's outstanding frontier back in the
+// queue and reports how many pages it queued. It is a no-op on a fresh run.
+func (c *Cloner) requeueUnfinished(ctx context.Context) int {
+	n := 0
+	for _, p := range c.front.unfinished() {
+		u, err := url.Parse(p.URL)
+		if err != nil {
+			c.logf("resume: dropping unreadable state entry %q: %v", p.URL, err)
+			continue
+		}
+		if c.enqueuePage(ctx, u, p.Depth) {
+			n++
+		}
+	}
+	return n
+}
+
 // enqueuePage offers a page URL to the frontier, honouring the visited set, the
 // depth cap, and the page budget. It reports whether the page was newly queued.
 func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
@@ -446,7 +485,10 @@ func (c *Cloner) enqueuePage(ctx context.Context, u *url.URL, depth int) bool {
 	if c.front.isVisited(key) {
 		return false
 	}
-	if !c.front.offer(key) {
+	// Recording the page as unfinished happens here, before the budget check
+	// below, so a page that was discovered but never started is still in the
+	// frontier the next run picks up (issue #36).
+	if !c.front.offer(key, pendingPage{URL: u.String(), Depth: depth}) {
 		return false
 	}
 	c.mu.Lock()

diff --git a/clone/cloner_test.go b/clone/cloner_test.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -423,3 +424,173 @@ func readAnyFile(t *testing.T, dir, name string) string {
 	})
 	return out
 }
+
+// chainSite is a five-page site linked in a chain, home to /p1 to /p2 to /p3 to
+// /p4. Nothing but following links finds the far end, so a resumed run that has
+// lost the frontier cannot reach it.
+func chainSite(t *testing.T) *httptest.Server {
+	t.Helper()
+	page := func(title, next string) string {
+		body := `<!doctype html><html><head><title>` + title + `</title></head><body><h1>` + title + `</h1>`
+		if next != "" {
+			body += `<a href="` + next + `">next</a>`
+		}
+		return body + `</body></html>`
+	}
+	links := map[string]string{
+		"/":   "/p1",
+		"/p1": "/p2",
+		"/p2": "/p3",
+		"/p3": "/p4",
+		"/p4": "",
+	}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("User-agent: *\nAllow: /\n"))
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		next, ok := links[r.URL.Path]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte(page(r.URL.Path, next)))
+	})
+	return httptest.NewServer(mux)
+}
+
+// TestCloneResumeFinishesTheCrawl guards issue #36. Only the visited set used to
+// be persisted, and the frontier was rebuilt purely by re-rendering pages and
+// following their links, which resume guarantees never happens. So an
+// interrupted crawl, restarted, found its seed already visited, queued nothing,
+// and exited successfully having done no work at all. The README promised the
+// opposite.
+func TestCloneResumeFinishesTheCrawl(t *testing.T) {
+	if testing.Short() {
+		t.Skip("resume test drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping resume test")
+	}
+
+	srv := chainSite(t)
+	defer srv.Close()
+	seed, _ := urlx.ParseSeed(srv.URL)
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
+	defer cancel()
+
+	// Stop the first run partway through, the way Ctrl-C would.
+	first := cfg
+	first.MaxPages = 2
+	res1, err := New(seed, first, t.Logf).Run(ctx)
+	if err != nil {
+		t.Fatalf("first run: %v", err)
+	}
+	if res1.Pages != 2 {
+		t.Fatalf("first run wrote %d pages, want 2", res1.Pages)
+	}
+
+	// Resume with no cap: it must pick up the frontier the first run left and
+	// walk the rest of the chain.
+	res2, err := New(seed, cfg, t.Logf).Run(ctx)
+	if err != nil {
+		t.Fatalf("resume run: %v", err)
+	}
+	if res2.Pages != 3 {
+		t.Errorf("resume wrote %d pages, want the 3 that were left", res2.Pages)
+	}
+
+	root := filepath.Join(out, seed.Hostname())
+	for _, p := range []string{"index.html", "p1/index.html", "p2/index.html", "p3/index.html", "p4/index.html"} {
+		if !fileExists(filepath.Join(root, p)) {
+			t.Errorf("%s was never written", p)
+		}
+	}
+
+	// Nothing is left over, so a third run has genuinely nothing to do.
+	res3, err := New(seed, cfg, t.Logf).Run(ctx)
+	if err != nil {
+		t.Fatalf("third run: %v", err)
+	}
+	if res3.Pages != 0 {
+		t.Errorf("third run wrote %d pages, want 0", res3.Pages)
+	}
+}
+
+// TestCloneResumeRetriesFailures covers the other half of issue #36: the
+// reporter asked for "a memory of what failed" so a later run could pick the
+// missing pages up. A page that errored is not marked visited, so it stays in
+// the frontier and a resumed run tries it again.
+func TestCloneResumeRetriesFailures(t *testing.T) {
+	if testing.Short() {
+		t.Skip("resume test drives Chrome; skipped under -short")
+	}
+	if _, ok := browser.LookChrome(); !ok {
+		t.Skip("no Chrome/Chromium found; skipping resume test")
+	}
+
+	var flaky atomic.Bool
+	mux := http.NewServeMux()
+	mux.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("User-agent: *\nAllow: /\n"))
+	})
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/":
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = w.Write([]byte(`<!doctype html><html><body><a href="/flaky">flaky</a></body></html>`))
+		case "/flaky":
+			// Down for the first run, back up for the second. The connection is
+			// dropped rather than answered with a 5xx because Chrome renders an
+			// error page happily; only a dead connection is a render failure.
+			if !flaky.Load() {
+				if hj, ok := w.(http.Hijacker); ok {
+					if conn, _, err := hj.Hijack(); err == nil {
+						_ = conn.Close()
+						return
+					}
+				}
+				http.Error(w, "down", http.StatusInternalServerError)
+				return
+			}
+			w.Header().Set("Content-Type", "text/html; charset=utf-8")
+			_, _ = w.Write([]byte(`<!doctype html><html><body><h1>back up</h1></body></html>`))
+		default:
+			http.NotFound(w, r)
+		}
+	})
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+	seed, _ := urlx.ParseSeed(srv.URL)
+
+	out := t.TempDir()
+	cfg := DefaultConfig()
+	cfg.OutDir = out
+	cfg.Settle = 300 * time.Millisecond
+
+	ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
+	defer cancel()
+
+	if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil {
+		t.Fatalf("first run: %v", err)
+	}
+	flakyPath := filepath.Join(out, seed.Hostname(), "flaky", "index.html")
+	if fileExists(flakyPath) {
+		t.Fatal("the flaky page should not have been written while it was down")
+	}
+
+	flaky.Store(true)
+	if _, err := New(seed, cfg, t.Logf).Run(ctx); err != nil {
+		t.Fatalf("resume run: %v", err)
+	}
+	if !fileExists(flakyPath) {
+		t.Error("resume should have retried the page that failed")
+	}
+}