diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53ce3ea..59723de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,20 @@ jobs: node-version: 20 - run: npm run validate + bench-smoke: + name: Benchmark harness smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + # Runs the two smallest corpora once each - enough to catch the harness + # bit-rotting, without publishing CI timings as if they were meaningful + # (shared runners are far too noisy for that). Real numbers come from + # running `npm run bench` locally; see docs/benchmarks/dataset-scale.md. + - run: node scripts/bench/run-benchmarks.mjs --sizes 12,100 --repeat 1 + validate-changelog: name: Changelog updated runs-on: ubuntu-latest @@ -58,4 +72,4 @@ jobs: echo "::error file=CHANGELOG.md::This PR modifies fixture files but the [Unreleased] section of CHANGELOG.md is empty." exit 1 fi - echo "CHANGELOG.md [Unreleased] section is populated — check passed." + echo "CHANGELOG.md [Unreleased] section is populated ๏ฟฝ check passed." diff --git a/.gitignore b/.gitignore index 3c45938..e782cb0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ *.log .DS_Store +.bench*/ diff --git a/README.md b/README.md index 36d0af4..96fba15 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ grydlock-testkit/ destinations.json scores.json scripts/validate-fixtures.mjs + scripts/bench/ + docs/benchmarks/dataset-scale.md transactions/ payment.xdr path_payment.xdr @@ -38,6 +40,18 @@ npm run validate Checks that every destination in destinations.json has a matching entry in scores.json, every score is an integer in 0-100, and every label is one of clean, suspicious, or malicious. +## Dataset Scale Benchmarks + +[docs/benchmarks/dataset-scale.md](docs/benchmarks/dataset-scale.md) answers how fixture loading should behave as the corpus grows past its current 12 entries. It compares five loading strategies at five corpus sizes (12 to 100,000) on startup latency, lookup latency, memory, and artifact size, and sets the thresholds at which the current raw-JSON layout stops being sufficient. + +Short version: nothing needs to change now, and the JSON files should stay the reviewable source of truth at every size - only what gets *published* changes as the corpus grows. + +Reproduce the measurements with: + +npm run bench + +That generates synthetic corpora under .bench/ (git-ignored, deterministic), measures each strategy in an isolated child process, and writes .bench/results.json and .bench/results.md. See scripts/bench/ for the harness and docs/benchmarks/dataset-scale.md for how to interpret the output. + ## How It's Used - grydlock-oracle-adapter loads scores.json in its StubOracle to return scores without a live backend diff --git a/docs/benchmarks/dataset-scale.md b/docs/benchmarks/dataset-scale.md new file mode 100644 index 0000000..4bb877f --- /dev/null +++ b/docs/benchmarks/dataset-scale.md @@ -0,0 +1,343 @@ +# Spike: dataset growth and fixture-loading strategies + +**Question:** how should fixture loading and lookup behave as the corpus grows from 12 entries to +hundreds or thousands? + +**Answer in one line:** the current two-file raw-JSON layout is comfortable to roughly **5,000 +entries**, needs a derived release artifact between **5,000 and 25,000**, and needs lazily-loaded +split packs beyond that. Nothing needs to change today. + +Everything below is reproducible with `npm run bench`. The harness lives in +[scripts/bench/](../../scripts/bench/); measurements were taken with the harness at the commit that +introduced this document. + +--- + +## TL;DR + +| Corpus size | What to do | Why | +|---|---|---| +| โ‰ค 1,000 | Keep `destinations.json` + `scores.json` exactly as they are | 3.9 ms cold load, 415 KB heap, 82 KB gzipped release. Every alternative is a rounding error's worth of gain against real added complexity. | +| 1,000 โ€“ 5,000 | Same, plus switch consumers from plain-object lookup to a `Map` index | Costs ~0.5 ms at load, removes prototype-inherited-key hazards, and cuts retained heap by ~3.5x. | +| 5,000 โ€“ 25,000 | Keep the JSON files as the reviewable source of truth; publish a **derived, minimal, gzipped artifact** alongside them | Halves the shipped bytes and the parse time by dropping the ~50% of each record that is review metadata the runtime never reads. | +| > 25,000 | Move to **lazily-loaded split packs** | Startup and memory stop depending on corpus size entirely: 1.6 ms and 10 KB at 100,000 entries, against 221 ms and 41 MB for the baseline. | + +The single most important number in this document: at 100,000 entries a cold extension session +(load the index, score 10 destinations) costs **220 ms and 41 MB** on the current strategy and +**20 ms and 389 KB** on split packs. + +--- + +## Method + +### Harness + +```bash +npm run bench # default sizes, median of 3 processes +node scripts/bench/run-benchmarks.mjs \ + --sizes 12,100,1000,10000,100000 --repeat 5 # what produced the tables below +node scripts/bench/generate-corpus.mjs --sizes 1000 # corpora only, no measurement +``` + +Results are written to `.bench/results.json` and `.bench/results.md`. `.bench/` is git-ignored: +the corpora are regenerated deterministically, so there is nothing to commit. + +| File | Role | +|---|---| +| `scripts/bench/generate-corpus.mjs` | Builds synthetic corpora at any size | +| `scripts/bench/strkey.mjs` | Mints valid synthetic `Gโ€ฆ` account IDs; seeded PRNG | +| `scripts/bench/strategies.mjs` | The five strategies, each `build()` + `load()` | +| `scripts/bench/measure.mjs` | Measures one (strategy, corpus) pair in an isolated process | +| `scripts/bench/run-benchmarks.mjs` | Drives the sweep, aggregates, renders the tables | + +### Corpora + +Sizes **12, 100, 1,000, 10,000, 100,000**. The 12-entry corpus is the real +`destinations.json` / `scores.json` copied verbatim, so the baseline row is what consumers load +today rather than a synthetic stand-in. Larger corpora keep all 12 real entries and top up with +synthetic ones drawn from the same label mix (34% clean / 25% suspicious / 41% malicious), the same +`risk_pattern` vocabulary, roughly 1-in-12 assets, and `notes` strings of realistic length โ€” notes +are the largest per-entry contributor to byte size, so short filler would have flattered every +artifact-size number. + +Generated account IDs are real Stellar strkeys (version byte, 32-byte payload, CRC16-XModem +checksum, base32) built from a seeded PRNG, so corpora are byte-identical across machines and runs. +Only the `Gโ€ฆ` public-key form is implemented, deliberately: nothing in this harness can emit +something that looks like a secret seed. + +### Measurement + +Each (size, strategy) cell is the **median of 5 independent `node --expose-gc` processes**. A fresh +process per cell keeps one strategy's module cache, JIT state, and heap out of the next one's +numbers. Within a process the phases run in a fixed order: + +1. **Cold load** on an untouched heap โ€” `heapUsed` before, `load()`, forced GC, `heapUsed` after. +2. **Cold session** โ€” the first 10 distinct lookups against the freshly loaded index, plus bytes + and shards pulled to answer them. +3. **Sustained lookups** โ€” 200,000 timed lookups cycling a 512-ID working set (hits), then the + same for misses, after a JIT warm-up pass. +4. **Warm loads** โ€” five further `load()` calls, median reported. + +The harness also asserts correctness on every cell: sampled IDs must resolve to their true score, +misses must resolve to `undefined`, and inherited keys (`constructor`, `toString`, `__proto__`, +`valueOf`) must resolve to `undefined`. Any failure is reported in the results. + +### Environment + +Node v22.23.0, win32/x64, AMD Ryzen 7 PRO 7730U, 15 GB RAM. **Absolute numbers are +machine-specific; the ratios and the scaling shapes are what the recommendation rests on.** The +1,000- and 10,000-entry sweeps were repeated on Node v20.20.2 (the version CI pins): every load +figure came out 10โ€“30% slower, and the ordering, the crossover points, and the per-entry constants +were unchanged. + +### What is *not* measured + +- **Real browser conditions.** These are Node numbers. A browser adds bundler overhead, and turns + split-pack shard reads into async `fetch`/IndexedDB round trips (see caveats). +- **Disk cold-start.** The OS page cache is warm across repetitions, so first-ever-read I/O is + excluded. That flatters every eager strategy equally. +- **Scoring correctness.** Out of scope here; that is `grydlock-research`'s job. + +--- + +## Strategies compared + +| # | Strategy | Artifact | Startup work | Lookup | +|---|---|---|---|---| +| 1 | **Raw JSON + object lookup** (baseline) | `destinations.json` + `scores.json`, verbatim | Read + `JSON.parse` both files | `scores[id]` | +| 2 | **Raw JSON + `Map` index** | identical to baseline | Parse both, then build a `Map` | `index.get(id)` | +| 3 | **Generated ES module** | one `fixtures.mjs` exporting packed rows | `import()` + build `Map` | `index.get(id)` | +| 4 | **Gzipped JSON archive** | one `fixtures.json.gz` of packed rows | Inflate, parse, build `Map` | `index.get(id)` | +| 5 | **Split packs, lazily loaded** | `manifest.json` + `packs/N.json` (1,000 entries/shard) | Read the manifest only | FNV-1a hash โ†’ shard, read shard on first touch, then `Map` | + +Strategies 3โ€“5 ship a **packed** record โ€” `[id, score, label, risk_pattern]` โ€” rather than the full +review record. That is a real difference, not an accounting trick: `notes`, `address`, `type`, and +the JSON key names exist for human review and account for roughly half of the raw bytes, and no +consumer reads them at lookup time. Where a table compares strategy 1 against 3โ€“5 on artifact size, +part of the gap is compression and part is dropping fields the runtime never touches; both are +things a release build gets to do, and neither requires changing the source of truth. + +Split packs place an ID by hashing it, so no global index has to be shipped or held in memory to +know which shard to fetch โ€” the manifest stays at 100 bytes for 1,000 entries and 3.9 KB for +100,000. + +--- + +## Results + +### Startup: cold load (ms) + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 1.78 | 1.78 | 3.85 | 23.81 | 220.51 | +| Raw JSON + Map index | 2.00 | 2.00 | 4.36 | 31.38 | 348.19 | +| Generated ES module | 2.50 | 2.19 | 3.40 | 15.65 | 159.30 | +| Gzipped JSON archive | 2.45 | 2.53 | 3.35 | 11.40 | 129.76 | +| Split packs, lazily loaded | 1.46 | 1.31 | 1.04 | 1.25 | 1.62 | + +Below 1,000 entries every strategy sits inside process-noise of every other. Past 1,000 the eager +strategies go linear at roughly **2.2 ยตs per entry** for the baseline; split packs stay flat because +startup only reads the manifest. + +### Memory: heap retained after load + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 8.4 KB | 41.2 KB | 414.7 KB | 3.9 MB | 41.3 MB | +| Raw JSON + Map index | 4.2 KB | 22.8 KB | 117.9 KB | 1.4 MB | 13.2 MB | +| Generated ES module | 18.4 KB | 40.5 KB | 249.9 KB | 2.5 MB | 23.5 MB | +| Gzipped JSON archive | 18.5 KB | 27.8 KB | 124.1 KB | 1.2 MB | 10.5 MB | +| Split packs, lazily loaded | 4.2 KB | 4.2 KB | 4.2 KB | 2.9 KB | 9.9 KB | + +The baseline retains the whole parsed `destinations` array โ€” every `notes` string included โ€” at +about **420 bytes per entry**. Discarding review metadata at load time and keeping only a `Map` of +what lookups need cuts that by ~3.5x for free; strategy 2 is the cheapest such change because it +keeps the artifact untouched. + +### Lookup: sustained, warm (ns/op) + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | miss @ 100,000 | +|---|---:|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 11 | 22 | 10 | 13 | 19 | 41 | +| Raw JSON + Map index | 18 | 17 | 22 | 38 | 79 | 66 | +| Generated ES module | 20 | 19 | 24 | 30 | 51 | 53 | +| Gzipped JSON archive | 15 | 17 | 21 | 28 | 47 | 50 | +| Split packs, lazily loaded | 136 | 143 | 148 | 155 | 721 | 232 | + +**Lookup latency is not a real constraint at any size measured.** Even the slowest cell is under a +microsecond, against a scoring path that already does XDR decoding and (in production) a network +call. The baseline's plain-object lookup is genuinely the fastest โ€” V8 dictionary-mode objects with +interned string keys beat `Map.get` here โ€” but a 60 ns difference cannot justify a decision when +startup differs by 200 ms. Split packs pay an FNV-1a hash over a 56-character key per lookup on top +of the `Map`, which is where their ~140 ns floor comes from. + +### Artifact size (KB) + +Gzipped total, which is what a release download and a browser transfer actually cost: + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 1.8 | 9.5 | 82.2 | 809.2 | 8,074.8 | +| Raw JSON + Map index | 1.8 | 9.5 | 82.2 | 809.2 | 8,074.8 | +| Generated ES module | 0.8 | 4.4 | 39.3 | 385.6 | 3,847.3 | +| Gzipped JSON archive | 0.6 | 4.2 | 39.1 | 385.5 | 3,848.2 | +| Split packs, lazily loaded | 0.6 | 3.9 | 37.3 | 372.4 | 3,721.9 | + +Bytes that must be fetched **before the first lookup can be answered** โ€” the number that matters for +extension startup and browser bundle size: + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 5.0 | 41.5 | 416.0 | 4,164.0 | 41,616.1 | +| Raw JSON + Map index | 5.0 | 41.5 | 416.0 | 4,164.0 | 41,616.1 | +| Generated ES module | 1.2 | 8.9 | 88.2 | 879.8 | 8,792.4 | +| Gzipped JSON archive | 0.6 | 4.2 | 39.0 | 385.3 | 3,847.0 | +| Split packs, lazily loaded | 0.1 | 0.1 | 0.1 | 0.4 | 3.9 | + +Raw JSON grows at **~416 bytes/entry uncompressed, ~81 bytes/entry gzipped**. A bundler that inlines +`destinations.json` into the extension pays the uncompressed column. + +### Cold session: load + 10 distinct lookups + +This is the metric that decides the recommendation. An extension does not sweep the corpus; it +scores a handful of destinations and then the MV3 service worker is torn down and has to do it all +again on the next wake-up. + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 1.9 ms | 1.9 ms | 3.9 ms | 23.9 ms | 220.6 ms | +| Raw JSON + Map index | 2.1 ms | 2.1 ms | 4.4 ms | 31.5 ms | 348.3 ms | +| Generated ES module | 2.6 ms | 2.3 ms | 3.5 ms | 15.8 ms | 159.4 ms | +| Gzipped JSON archive | 2.6 ms | 2.6 ms | 3.4 ms | 11.6 ms | 129.9 ms | +| Split packs, lazily loaded | 3.1 ms | 2.8 ms | 3.2 ms | 16.2 ms | 19.7 ms | + +Heap held at the end of that session: + +| Strategy | 12 | 100 | 1,000 | 10,000 | 100,000 | +|---|---:|---:|---:|---:|---:| +| Raw JSON + object lookup (baseline) | 9 KB | 41 KB | 415 KB | 3.9 MB | 41.3 MB | +| Raw JSON + Map index | 5 KB | 24 KB | 119 KB | 1.4 MB | 13.2 MB | +| Generated ES module | 20 KB | 42 KB | 251 KB | 2.5 MB | 23.5 MB | +| Gzipped JSON archive | 19 KB | 29 KB | 125 KB | 1.2 MB | 10.5 MB | +| Split packs, lazily loaded | 15 KB | 18 KB | 42 KB | 354 KB | 389 KB | + +Split packs pulled **604 KB across 10 shards** to answer that 10-lookup session at 100,000 entries, +and **549 KB across 9 shards** at 10,000 โ€” essentially the same cost, because lazy cost scales with +`working set x shard size`, not with corpus size. That is also why split packs *lose* at 10,000: the +whole corpus is only 10 shards, so a 10-lookup session touches nearly all of it and pays the hashing +overhead for nothing. Shard size (`SHARD_SIZE` in `strategies.mjs`, currently 1,000) is the knob: +smaller shards lower session cost and raise file count and request count. + +--- + +## Comparison matrix + +Measured columns are at 10,000 entries. Qualitative columns are judgements, marked as such. + +| Strategy | Startup | Lookup | Memory | Ship size | Release complexity | Portability | Developer experience | +|---|---|---|---|---|---|---|---| +| **Raw JSON + object lookup** (baseline) | 23.8 ms | 13 ns | 3.9 MB | 809 KB gz | **None** โ€” the files are the release | **Best** โ€” any language, any runtime, `curl` and read | **Best** โ€” reviewable diffs, no build step, `npm run validate` is the whole toolchain | +| **Raw JSON + `Map` index** | 31.4 ms | 38 ns | 1.4 MB | 809 KB gz | **None** โ€” consumer-side change only | Same as baseline | Same as baseline; ~10 lines in each consumer | +| **Generated ES module** | 15.7 ms | 30 ns | 2.5 MB | 386 KB gz | Low โ€” one generator, one checked artifact | **Poor** โ€” JS consumers only; opaque to non-JS tooling | Bundler-friendly and tree-shakeable, but a generated file to keep in sync and to review | +| **Gzipped JSON archive** | 11.4 ms | 28 ns | 1.2 MB | 386 KB gz | Low โ€” one generator; release asset, not a source file | Good โ€” gzip is universal, though browsers need `DecompressionStream` or a bundler step | Opaque in diffs; the JSON files stay the reviewable source | +| **Split packs, lazily loaded** | **1.2 ms** | 155 ns | **2.9 KB** | 372 KB gz | **High** โ€” manifest, shard layout, hash function, and cache invalidation all become a contract | Fair โ€” plain JSON files, but every consumer must implement the manifest + hash | Worst โ€” a fixture change reshuffles shards; needs a partial-fetch-capable consumer | + +--- + +## Findings + +1. **Nothing is wrong today.** At 12 entries the baseline loads in 1.8 ms and holds 8 KB. The + spike found no reason to change anything now. +2. **Startup, not lookup, is the constraint.** Lookup stays under 1 ยตs everywhere; cold load spans + 1.8 ms to 348 ms across the sweep. Optimising lookup would be optimising the wrong axis. +3. **The baseline's cost is dominated by data the runtime never reads.** `notes`, `address`, `type`, + and JSON key names are roughly half the bytes and most of the retained heap. Any derived + artifact โ€” module, gzip, or packs โ€” gets that back without touching the source of truth. +4. **Retained heap is the baseline's worst axis**, at ~420 bytes/entry. It reaches 3.9 MB at 10,000 + and 41 MB at 100,000, which is a real problem for an MV3 service worker. +5. **MV3 amplifies startup cost.** The extension's service worker is evicted and respawned + routinely, so cold load is paid many times per browsing session, not once per install. A 24 ms + parse at 10,000 entries is not a one-off. +6. **Lazy loading only pays when the corpus is much larger than the working set.** Split packs lose + at โ‰ค 10,000 entries and win by 11x at 100,000. Adopting them early would be a net loss. +7. **Plain-object lookup is not prototype-safe.** The harness records this on every baseline run: + `scores['constructor']` returns a function rather than `undefined`. Stellar strkeys and + `CODE:ISSUER` asset IDs cannot collide with `Object.prototype` keys, so this is a latent + robustness issue rather than a live bug โ€” but it is free to remove by indexing into a `Map`. +8. **Compression does most of the work that split packs do, at a fraction of the complexity.** + Gzipped JSON is within 4% of split packs on total ship size and beats every other eager strategy + on startup and memory. + +--- + +## Recommendation + +**Keep the current layout. Do not build split packs now.** + +`destinations.json` + `scores.json` should stay the reviewable source of truth at every size โ€” the +labelling rubric, the changelog discipline, and the reviewer checklist in `CONTRIBUTING.md` all +depend on a fixture change being a readable diff. What should change as the corpus grows is what +gets *published*, not what gets *reviewed*. + +Staged plan: + +1. **Now (โ‰ค 1,000 entries):** no change. +2. **Consumer-side, whenever convenient:** have `StubOracle` build a `Map` at load and drop the + parsed `destinations` array once the index exists. Small change, ~3.5x less retained heap, + removes the prototype-key hazard. Worth doing before the corpus reaches 1,000. +3. **At ~5,000 entries:** add a release build step emitting a packed, gzipped artifact alongside the + JSON files, and have consumers prefer it. Halves both bytes shipped and parse time. +4. **At ~25,000 entries:** revisit split packs, with shard size tuned against the consumer's real + working set โ€” and re-run this harness first, since the crossover point depends on that working + set far more than on the corpus size. + +### Thresholds + +Budgets chosen so that the fixture layer stays an insignificant fraction of a signing-path +interaction. Re-run `npm run bench` when the corpus crosses a size row, and treat a breached budget +as the trigger for the next stage. + +| Metric | Budget | Baseline breaches it at | Notes | +|---|---|---|---| +| Cold load (Node) | 10 ms | **~5,000 entries** | Extrapolated at ~2.2 ยตs/entry; 3.9 ms measured at 1,000, 23.8 ms at 10,000 | +| Cold load (Node) | 50 ms | ~20,000 entries | Hard ceiling for an MV3 worker that respawns often | +| Retained heap | 5 MB | **~12,000 entries** | ~420 bytes/entry; 3.9 MB measured at 10,000 | +| Bytes before first lookup | 1 MB uncompressed | **~2,500 entries** | ~416 bytes/entry; matters most for a bundler that inlines the JSON | +| Release artifact | 5 MB gzipped | ~60,000 entries | ~81 bytes/entry gzipped | +| Warm lookup | 1 ยตs | not breached at 100,000 | Not a practical constraint at any size studied | + +The **first** budget the current approach breaches is bytes-before-first-lookup at ~2,500 entries +for a bundler-inlining consumer, then cold load at ~5,000. Stated plainly: **the current approach +becomes insufficient somewhere between 2,500 and 5,000 entries, and stage 3 above is the response.** + +--- + +## Caveats + +- **Node, not a browser.** Split packs read shards synchronously here; in a browser each cold shard + is an async `fetch` or IndexedDB read, adding milliseconds and making `getScore` unavoidably + async. The adapter's `IOracle` is already async, so the interface survives โ€” but a browser + measurement should precede any split-pack adoption. +- **Uniform access assumed.** The 512-ID working set is spread evenly across the corpus. Real + lookups are skewed toward a small set of hot destinations, which favours lazy loading more than + these numbers show, and would favour it further with an LRU shard cache. +- **Synthetic entries above 12.** Field shapes and note lengths mirror the real corpus, but a future + schema change (extra fields, longer notes, richer metadata) shifts every per-entry constant here. + Re-run rather than re-reading these tables after a schema change. +- **Single machine.** The ratios are stable; the absolute milliseconds are not portable. +- **Gzip only.** Brotli would likely shave a further 15โ€“20% off the artifact-size column and is + worth measuring if ship size ever becomes the binding constraint. + +## Cross-repository impact + +- **`grydlock-testkit`:** no change now. Stage 3 adds a release build step and a second release + asset; the JSON files stay canonical. +- **`grydlock-oracle-adapter`:** `StubOracle` currently vendors the JSON and indexes by plain-object + access. The `Map` change (stage 2) belongs there and is worth doing regardless of corpus size. + The consumer contract test should keep asserting against `destinations.json`, since that stays the + source of truth under every stage. +- **`grydlock-extension`:** the beneficiary of every threshold here. Its MV3 service worker + lifecycle is why startup latency is weighted above lookup latency throughout. +- **`grydlock-research`:** eager loading is correct for research โ€” it sweeps the whole corpus, so + lazy loading is strictly worse. If split packs are ever adopted, research should keep loading a + single concatenated artifact. diff --git a/package.json b/package.json index f220cbc..41bd881 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,8 @@ "license": "MIT", "private": true, "scripts": { - "validate": "node scripts/validate-fixtures.mjs" + "validate": "node scripts/validate-fixtures.mjs", + "bench": "node scripts/bench/run-benchmarks.mjs", + "bench:corpus": "node scripts/bench/generate-corpus.mjs" } } diff --git a/scripts/bench/generate-corpus.mjs b/scripts/bench/generate-corpus.mjs new file mode 100644 index 0000000..e075fd1 --- /dev/null +++ b/scripts/bench/generate-corpus.mjs @@ -0,0 +1,215 @@ +/** + * Generate synthetic fixture corpora at several sizes so fixture-loading + * strategies can be benchmarked well past the 12 entries that exist today. + * + * Usage: + * node scripts/bench/generate-corpus.mjs [--sizes 12,100,1000,10000] [--out .bench] + * + * The size that matches the real corpus (12) is *copied* from the repository + * root rather than synthesised, so the baseline measurement reflects the + * fixtures consumers actually load today. + * + * Output is deterministic: the same `--sizes` always produce byte-identical + * corpora, so numbers from two runs (or two machines) are comparable. + */ + +import { mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +import { encodeAccountId, seededRandom } from './strkey.mjs'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); + +export const DEFAULT_SIZES = [12, 100, 1000, 10000]; + +/** Label mix, chosen to stay close to the real corpus (4/3/5 of 12). */ +const LABEL_MIX = [ + { label: 'clean', weight: 0.34, scoreMin: 0, scoreMax: 25 }, + { label: 'suspicious', weight: 0.25, scoreMin: 40, scoreMax: 70 }, + { label: 'malicious', weight: 0.41, scoreMin: 75, scoreMax: 100 } +]; + +const RISK_PATTERNS = { + clean: ['none', 'adversarial-clean'], + suspicious: ['pass-through', 'scam-trustline', 'cold-start', 'sponsored-mule'], + malicious: ['sweep', 'phishing-drainer', 'rug-pull', 'signer-takeover', 'memo-impersonation'] +}; + +/** + * Note templates. Length matters: `notes` is the single largest contributor to + * per-entry byte size, so synthetic entries must be about as verbose as the + * hand-written ones or every artifact-size number would be optimistic. + */ +const NOTE_TEMPLATES = { + clean: [ + 'Established testnet wallet with regular payment history; no red flags.', + 'Long-lived wallet, low transaction velocity, no shared funding source with flagged accounts.', + 'Holds a USDC trustline against a long-lived issuer; normal trading activity.', + 'Fixture stand-in for a well-known, long-lived asset issuer with broad distribution.' + ], + suspicious: [ + 'Funded seconds before a large outgoing payment burst; pattern consistent with a pass-through wallet.', + 'Shares a funding source with two destinations already labelled malicious in this corpus.', + 'Holds a trustline to a flagged scam asset; no other activity on the account.', + 'Transaction velocity is unusual for the account age; no direct evidence of fraud yet.' + ], + malicious: [ + 'Path-payment sweep pattern: drains newly funded wallets within seconds of receiving deposits.', + "Repeated small 'test' payments precede a large drain; matches a known phishing-drainer shape.", + 'Destination of multiple change-trust + immediate max-sell patterns typical of rug-pull collection wallets.', + 'Issuer of a scam asset with no distributed supply outside the issuing account.' + ] +}; + +const ASSET_CODES = ['SCAM', 'RUGX', 'FREEB', 'AIRDR', 'MOONX', 'YLDZ', 'PUMPZ', 'GIFTX']; + +/** Roughly 1 in 12 entries is an asset, mirroring the real corpus. */ +const ASSET_RATIO = 1 / 12; + +function pick(rng, list) { + return list[Math.floor(rng() * list.length) % list.length]; +} + +function pickLabel(rng) { + const roll = rng(); + let cumulative = 0; + for (const bucket of LABEL_MIX) { + cumulative += bucket.weight; + if (roll < cumulative) return bucket; + } + return LABEL_MIX[LABEL_MIX.length - 1]; +} + +function randomAccountId(rng) { + const payload = new Uint8Array(32); + for (let i = 0; i < 32; i++) { + payload[i] = Math.floor(rng() * 256); + } + return encodeAccountId(payload); +} + +/** + * Build a corpus of `size` destinations plus its matching score map. + * + * @param {number} size + * @param {number} seed + * @returns {{ destinations: object[], scores: Record }} + */ +export function buildCorpus(size, seed = 0x51ee) { + const rng = seededRandom(seed ^ size); + const destinations = []; + const scores = {}; + const seen = new Set(); + + while (destinations.length < size) { + const bucket = pickLabel(rng); + const account = randomAccountId(rng); + const isAsset = rng() < ASSET_RATIO; + const score = + bucket.scoreMin + Math.floor(rng() * (bucket.scoreMax - bucket.scoreMin + 1)); + + const assetCode = pick(rng, ASSET_CODES); + const entry = isAsset + ? { + id: `${assetCode}:${account}`, + type: 'asset', + asset_code: assetCode, + asset_issuer: account, + label: bucket.label, + risk_pattern: pick(rng, RISK_PATTERNS[bucket.label]), + notes: pick(rng, NOTE_TEMPLATES[bucket.label]) + } + : { + id: account, + type: 'account', + address: account, + label: bucket.label, + risk_pattern: pick(rng, RISK_PATTERNS[bucket.label]), + notes: pick(rng, NOTE_TEMPLATES[bucket.label]) + }; + + if (seen.has(entry.id)) continue; + seen.add(entry.id); + + destinations.push(entry); + scores[entry.id] = score; + } + + return { destinations, scores }; +} + +/** + * Materialise one corpus on disk in the same two-file shape consumers read + * today (`destinations.json` + `scores.json`). + * + * @param {number} size + * @param {string} outRoot absolute path of the benchmark working directory + * @returns {{ size: number, dir: string, synthetic: boolean }} + */ +export function writeCorpus(size, outRoot) { + const dir = join(outRoot, `corpus-${size}`); + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + + const realDestinations = JSON.parse(readFileSync(join(ROOT, 'destinations.json'), 'utf-8')); + const realScores = JSON.parse(readFileSync(join(ROOT, 'scores.json'), 'utf-8')); + const realSize = realDestinations.destinations.length; + + let payload; + let synthetic; + + if (size === realSize) { + // Baseline: the fixtures consumers load today, verbatim. + payload = { destinations: realDestinations.destinations, scores: realScores }; + synthetic = false; + } else if (size < realSize) { + throw new Error(`corpus size ${size} is below the real corpus size (${realSize})`); + } else { + // Larger corpora keep every real entry so lookups against known IDs stay + // valid, then top up with synthetic entries. + const generated = buildCorpus(size - realSize); + payload = { + destinations: [...realDestinations.destinations, ...generated.destinations], + scores: { ...realScores, ...generated.scores } + }; + synthetic = true; + } + + writeFileSync( + join(dir, 'destinations.json'), + JSON.stringify({ destinations: payload.destinations }, null, 2) + '\n' + ); + writeFileSync(join(dir, 'scores.json'), JSON.stringify(payload.scores, null, 2) + '\n'); + + return { size: payload.destinations.length, dir, synthetic }; +} + +function parseArgs(argv) { + const args = { sizes: DEFAULT_SIZES, out: '.bench' }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--sizes') { + args.sizes = argv[++i] + .split(',') + .map((s) => Number.parseInt(s.trim(), 10)) + .filter((n) => Number.isInteger(n) && n > 0); + } else if (argv[i] === '--out') { + args.out = argv[++i]; + } + } + return args; +} + +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isMain) { + const args = parseArgs(process.argv.slice(2)); + const outRoot = join(ROOT, args.out); + mkdirSync(outRoot, { recursive: true }); + + for (const size of args.sizes) { + const result = writeCorpus(size, outRoot); + const kind = result.synthetic ? 'synthetic' : 'real (baseline)'; + console.log(`corpus-${result.size}: ${kind} -> ${result.dir}`); + } +} diff --git a/scripts/bench/measure.mjs b/scripts/bench/measure.mjs new file mode 100644 index 0000000..f0f57d9 --- /dev/null +++ b/scripts/bench/measure.mjs @@ -0,0 +1,182 @@ +/** + * Measure one (strategy, corpus) pair in an isolated process. + * + * Run by `run-benchmarks.mjs`, never directly in the normal workflow: + * + * node --expose-gc scripts/bench/measure.mjs + * + * A fresh process per measurement keeps the module cache, the JIT state and + * the heap from one strategy out of the next one's numbers. Results are + * printed to stdout as a single JSON line. + * + * Phase order is deliberate: + * 1. memory + cold load, on an untouched heap + * 2. lookups, against that same loaded index + * 3. repeated loads, for a warm startup figure + */ + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { performance } from 'node:perf_hooks'; + +import { getStrategy } from './strategies.mjs'; + +const [strategyId, artifactDir, corpusDir] = process.argv.slice(2); + +/** Lookups timed per phase. Large enough to swamp timer resolution. */ +const LOOKUP_OPS = 200_000; +/** Distinct IDs cycled through, to defeat single-entry caching effects. */ +const SAMPLE_SIZE = 512; +/** Warm load repetitions after the cold one. */ +const WARM_LOADS = 5; +/** + * Distinct destinations a single extension session is assumed to score. A + * user signing a few transactions touches a handful of addresses, not the + * whole corpus - the gap between the two is what lazy loading trades on. + */ +const SESSION_LOOKUPS = 10; + +function gc() { + if (typeof global.gc === 'function') { + global.gc(); + global.gc(); + } +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = sorted.length >> 1; + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +/** Deterministic sample so every strategy is asked for the same IDs. */ +function sampleIds(ids, count) { + const step = Math.max(1, Math.floor(ids.length / count)); + const out = []; + for (let i = 0; i < ids.length && out.length < count; i += step) { + out.push(ids[i]); + } + return out; +} + +function timeLookups(handle, ids) { + let checksum = 0; + const start = performance.now(); + for (let i = 0; i < LOOKUP_OPS; i++) { + const value = handle.lookup(ids[i % ids.length]); + checksum += value === undefined ? 0 : value; + } + const elapsedMs = performance.now() - start; + return { nsPerOp: (elapsedMs * 1e6) / LOOKUP_OPS, checksum }; +} + +async function main() { + const strategy = getStrategy(strategyId); + + const destinations = JSON.parse( + readFileSync(join(corpusDir, 'destinations.json'), 'utf-8') + ).destinations; + const scores = JSON.parse(readFileSync(join(corpusDir, 'scores.json'), 'utf-8')); + const allIds = destinations.map((d) => d.id); + const hitIds = sampleIds(allIds, SAMPLE_SIZE); + const missIds = hitIds.map((id) => `MISS${id.slice(4)}`); + + // Drop the corpus we only needed to pick query IDs, so it is not counted as + // the strategy's memory. + destinations.length = 0; + + // --- Phase 1: cold load, on a clean heap ------------------------------- + gc(); + const heapBefore = process.memoryUsage().heapUsed; + const coldStart = performance.now(); + const handle = await strategy.load(artifactDir); + const coldLoadMs = performance.now() - coldStart; + gc(); + const heapAfterLoad = process.memoryUsage().heapUsed; + + // --- Phase 2: cold session probe --------------------------------------- + // What one extension session actually costs: a handful of distinct + // destinations scored against a freshly loaded index. This is the number + // that decides whether lazy loading pays off, because a lazy strategy only + // wins while the working set stays small relative to the corpus. + const sessionIds = sampleIds(allIds, SESSION_LOOKUPS); + const firstLookupStart = performance.now(); + const firstValue = handle.lookup(sessionIds[0]); + const firstLookupUs = (performance.now() - firstLookupStart) * 1000; + + for (let i = 1; i < sessionIds.length; i++) { + handle.lookup(sessionIds[i]); + } + const sessionMs = coldLoadMs + (performance.now() - firstLookupStart); + const sessionTouchedBytes = + typeof handle.touchedBytes === 'function' ? handle.touchedBytes() : null; + const sessionShards = typeof handle.loadedShards === 'function' ? handle.loadedShards() : null; + gc(); + const heapAfterSession = process.memoryUsage().heapUsed; + + // --- Phase 3: sustained lookup throughput ------------------------------- + // Warm the JIT before timing. + timeLookups(handle, hitIds.slice(0, 32)); + + const hit = timeLookups(handle, hitIds); + + // Snapshot before the miss sweep: a miss hashes to an arbitrary shard, so + // sweeping misses would pull the entire corpus into a lazy strategy and + // hide the whole point of loading lazily. + gc(); + const heapAfterHits = process.memoryUsage().heapUsed; + const touchedAfterHits = typeof handle.touchedBytes === 'function' ? handle.touchedBytes() : null; + const shardsAfterHits = typeof handle.loadedShards === 'function' ? handle.loadedShards() : null; + + const miss = timeLookups(handle, missIds); + + // --- Correctness ------------------------------------------------------- + const wrong = hitIds.filter((id) => handle.lookup(id) !== scores[id]); + const missesClean = missIds.every((id) => handle.lookup(id) === undefined); + // A plain-object index answers `constructor` and `toString` with inherited + // values instead of `undefined`; a Map does not. + const prototypeSafe = ['constructor', 'toString', '__proto__', 'valueOf'].every( + (key) => handle.lookup(key) === undefined + ); + + // --- Phase 4: warm loads ------------------------------------------------ + const warmLoadMs = []; + for (let i = 0; i < WARM_LOADS; i++) { + const start = performance.now(); + // eslint-disable-next-line no-await-in-loop -- sequential timing is the point + const reloaded = await strategy.load(artifactDir); + warmLoadMs.push(performance.now() - start); + if (reloaded.size !== handle.size) { + throw new Error('reload produced a different corpus size'); + } + } + + process.stdout.write( + JSON.stringify({ + strategy: strategyId, + entries: handle.size, + coldLoadMs, + warmLoadMs: median(warmLoadMs), + firstLookupUs, + sessionMs, + sessionLookups: SESSION_LOOKUPS, + sessionTouchedBytes, + sessionShards, + heapAfterSessionBytes: heapAfterSession - heapBefore, + hitLookupNs: hit.nsPerOp, + missLookupNs: miss.nsPerOp, + heapAfterLoadBytes: heapAfterLoad - heapBefore, + heapAfterHitsBytes: heapAfterHits - heapBefore, + lazyTouchedBytes: touchedAfterHits, + loadedShards: shardsAfterHits, + correct: wrong.length === 0 && missesClean && firstValue === scores[hitIds[0]], + prototypeSafe, + checksum: hit.checksum + }) + ); +} + +main().catch((err) => { + console.error(err.stack || String(err)); + process.exit(1); +}); diff --git a/scripts/bench/run-benchmarks.mjs b/scripts/bench/run-benchmarks.mjs new file mode 100644 index 0000000..2ce3106 --- /dev/null +++ b/scripts/bench/run-benchmarks.mjs @@ -0,0 +1,218 @@ +/** + * Fixture-loading benchmark harness. + * + * Usage: + * node scripts/bench/run-benchmarks.mjs + * node scripts/bench/run-benchmarks.mjs --sizes 12,100,1000,10000,100000 + * node scripts/bench/run-benchmarks.mjs --strategies raw-json,split-packs --repeat 5 + * + * For every (corpus size, strategy) pair it builds the artifact a release + * would ship, then measures load and lookup cost in a fresh child process + * (`measure.mjs`). Results are written to `.bench/results.json` and + * `.bench/results.md`; `.bench/` is git-ignored. + * + * Everything is deterministic apart from the timings themselves, and each + * timing is the median of `--repeat` independent processes. + */ + +import { execFileSync } from 'node:child_process'; +import { cpus, totalmem } from 'node:os'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { DEFAULT_SIZES, writeCorpus } from './generate-corpus.mjs'; +import { STRATEGIES, getStrategy } from './strategies.mjs'; + +const ROOT = fileURLToPath(new URL('../..', import.meta.url)); +const MEASURE = fileURLToPath(new URL('./measure.mjs', import.meta.url)); + +function parseArgs(argv) { + const args = { + sizes: DEFAULT_SIZES, + strategies: STRATEGIES.map((s) => s.id), + repeat: 3, + out: '.bench' + }; + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + if (flag === '--sizes') { + args.sizes = argv[++i] + .split(',') + .map((s) => Number.parseInt(s.trim(), 10)) + .filter((n) => Number.isInteger(n) && n > 0); + } else if (flag === '--strategies') { + args.strategies = argv[++i].split(',').map((s) => s.trim()); + } else if (flag === '--repeat') { + args.repeat = Math.max(1, Number.parseInt(argv[++i], 10)); + } else if (flag === '--out') { + args.out = argv[++i]; + } else { + throw new Error(`unknown flag "${flag}"`); + } + } + return args; +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = sorted.length >> 1; + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function measureOnce(strategyId, artifactDir, corpusDir) { + const stdout = execFileSync( + process.execPath, + ['--expose-gc', MEASURE, strategyId, artifactDir, corpusDir], + { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'inherit'] } + ); + return JSON.parse(stdout); +} + +const NUMERIC_KEYS = [ + 'coldLoadMs', + 'warmLoadMs', + 'firstLookupUs', + 'sessionMs', + 'hitLookupNs', + 'missLookupNs', + 'heapAfterLoadBytes', + 'heapAfterSessionBytes', + 'heapAfterHitsBytes' +]; + +function aggregate(runs) { + const out = { ...runs[0] }; + for (const key of NUMERIC_KEYS) { + out[key] = median(runs.map((r) => r[key])); + } + out.correct = runs.every((r) => r.correct); + out.prototypeSafe = runs.every((r) => r.prototypeSafe); + delete out.checksum; + return out; +} + +const kb = (bytes) => (bytes / 1024).toFixed(1); +const ms = (value) => value.toFixed(3); +const ns = (value) => value.toFixed(1); + +function toMarkdown(report) { + const lines = []; + lines.push('# Fixture-loading benchmark results'); + lines.push(''); + lines.push(`Generated by \`npm run bench\` on ${report.environment.date}.`); + lines.push(''); + lines.push( + `Node ${report.environment.node} | ${report.environment.platform}/${report.environment.arch} | ` + + `${report.environment.cpu} | ${report.environment.totalMemGb} GB RAM | ` + + `${report.repeat === 1 ? '1 process' : `median of ${report.repeat} processes`} per cell` + ); + lines.push(''); + + for (const size of report.sizes) { + const rows = report.results.filter((r) => r.entries === size); + if (rows.length === 0) continue; + lines.push(`## ${size.toLocaleString('en-US')} entries`); + lines.push(''); + lines.push( + '| Strategy | Cold load (ms) | Warm load (ms) | Hit lookup (ns) | Miss lookup (ns) | ' + + 'Heap after load (KB) | Startup bytes | Total bytes | Total gzip |' + ); + lines.push('|---|---:|---:|---:|---:|---:|---:|---:|---:|'); + for (const r of rows) { + lines.push( + `| ${r.label} | ${ms(r.coldLoadMs)} | ${ms(r.warmLoadMs)} | ${ns(r.hitLookupNs)} | ` + + `${ns(r.missLookupNs)} | ${kb(r.heapAfterLoadBytes)} | ${kb(r.artifact.initialBytes)} | ` + + `${kb(r.artifact.totalBytes)} | ${kb(r.artifact.totalGzipBytes)} |` + ); + } + lines.push(''); + lines.push( + `Cold session (load + ${rows[0].sessionLookups} distinct lookups), which is what one ` + + 'extension session actually pays:' + ); + lines.push(''); + lines.push('| Strategy | Session (ms) | Heap after session (KB) | Bytes pulled | Shards pulled |'); + lines.push('|---|---:|---:|---:|---:|'); + for (const r of rows) { + lines.push( + `| ${r.label} | ${ms(r.sessionMs)} | ${kb(r.heapAfterSessionBytes)} | ` + + `${r.sessionTouchedBytes === null ? 'all' : kb(r.sessionTouchedBytes) + ' KB'} | ` + + `${r.sessionShards === null ? 'n/a' : r.sessionShards} |` + ); + } + lines.push(''); + } + + lines.push('## Notes'); + lines.push(''); + for (const r of report.results) { + if (!r.correct) lines.push(`- ${r.label} @ ${r.entries}: FAILED the correctness check.`); + if (!r.prototypeSafe) { + lines.push( + `- ${r.label} @ ${r.entries}: not prototype-safe - inherited keys such as \`constructor\` resolve to a value instead of \`undefined\`.` + ); + } + if (r.lazyTouchedBytes !== null) { + lines.push( + `- ${r.label} @ ${r.entries}: pulled ${kb(r.lazyTouchedBytes)} KB across ${r.loadedShards} shard(s) to answer a 512-ID working set (vs ${kb(r.sessionTouchedBytes)} KB / ${r.sessionShards} shard(s) for a ${r.sessionLookups}-ID session); first lookup cost ${r.firstLookupUs.toFixed(1)} us.` + ); + } + } + lines.push(''); + return lines.join('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const outRoot = join(ROOT, args.out); + mkdirSync(outRoot, { recursive: true }); + + const results = []; + const sizes = []; + + for (const requested of args.sizes) { + const corpus = writeCorpus(requested, outRoot); + sizes.push(corpus.size); + console.log(`\ncorpus: ${corpus.size} entries (${corpus.synthetic ? 'synthetic' : 'real baseline'})`); + + for (const strategyId of args.strategies) { + const strategy = getStrategy(strategyId); + const artifactDir = join(outRoot, `artifacts-${corpus.size}`, strategyId); + const artifact = strategy.build(corpus.dir, artifactDir); + + const runs = []; + for (let i = 0; i < args.repeat; i++) { + runs.push(measureOnce(strategyId, artifactDir, corpus.dir)); + } + const result = { ...aggregate(runs), label: strategy.label, artifact }; + results.push(result); + + console.log( + ` ${strategy.label.padEnd(36)} load ${ms(result.coldLoadMs).padStart(8)} ms | ` + + `hit ${ns(result.hitLookupNs).padStart(7)} ns | heap ${kb(result.heapAfterLoadBytes).padStart(9)} KB | ` + + `ship ${kb(result.artifact.totalGzipBytes).padStart(8)} KB gz${result.correct ? '' : ' [INCORRECT]'}` + ); + } + } + + const report = { + environment: { + date: new Date().toISOString().slice(0, 10), + node: process.version, + platform: process.platform, + arch: process.arch, + cpu: cpus()[0]?.model.trim() ?? 'unknown', + totalMemGb: Math.round(totalmem() / 1024 ** 3) + }, + repeat: args.repeat, + sizes, + results + }; + + writeFileSync(join(outRoot, 'results.json'), JSON.stringify(report, null, 2) + '\n'); + writeFileSync(join(outRoot, 'results.md'), toMarkdown(report)); + console.log(`\nWrote ${join(args.out, 'results.json')} and ${join(args.out, 'results.md')}`); +} + +main(); diff --git a/scripts/bench/strategies.mjs b/scripts/bench/strategies.mjs new file mode 100644 index 0000000..d9c73ee --- /dev/null +++ b/scripts/bench/strategies.mjs @@ -0,0 +1,278 @@ +/** + * Fixture-loading strategies under evaluation. + * + * Every strategy exposes the same two-phase contract: + * + * build(corpusDir, artifactDir) -> artifact metadata (what a release ships) + * load(artifactDir) -> { lookup(id), touchedBytes() } + * + * `load()` is what a consumer pays at startup; `lookup(id)` is what it pays + * per scored destination. Keeping the shape identical across strategies is + * what makes the numbers comparable. + * + * `lookup()` is synchronous everywhere, including for the lazily-loaded split + * packs, because Node consumers can read a shard synchronously. A browser + * consumer would pay an async `fetch` there instead โ€” see the report for how + * that changes the picture. + */ + +import { + cpSync, + mkdirSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { gzipSync } from 'node:zlib'; + +/** Entries per shard for the split-pack strategy. */ +export const SHARD_SIZE = 1000; + +function readCorpus(corpusDir) { + const destinations = JSON.parse( + readFileSync(join(corpusDir, 'destinations.json'), 'utf-8') + ).destinations; + const scores = JSON.parse(readFileSync(join(corpusDir, 'scores.json'), 'utf-8')); + return { destinations, scores }; +} + +function freshDir(dir) { + rmSync(dir, { recursive: true, force: true }); + mkdirSync(dir, { recursive: true }); + return dir; +} + +/** + * Summarise every file in an artifact directory, on disk and gzipped. + * `initial` names the files a consumer must fetch before its first lookup; + * everything else is deferred. + */ +function measureArtifact(dir, initialFiles) { + const files = readdirSync(dir, { recursive: true, encoding: 'utf-8' }) + .filter((name) => statSync(join(dir, name)).isFile()) + .map((name) => { + const buf = readFileSync(join(dir, name)); + return { name: name.split('\\').join('/'), bytes: buf.length, gzipBytes: gzipSync(buf).length }; + }); + + const initial = new Set(initialFiles); + const sum = (list, key) => list.reduce((acc, f) => acc + f[key], 0); + const initialList = files.filter((f) => initial.has(f.name)); + + return { + fileCount: files.length, + totalBytes: sum(files, 'bytes'), + totalGzipBytes: sum(files, 'gzipBytes'), + initialBytes: sum(initialList, 'bytes'), + initialGzipBytes: sum(initialList, 'gzipBytes') + }; +} + +/** FNV-1a, used to place an ID in a shard without shipping a global index. */ +export function shardOf(id, shardCount) { + let hash = 0x811c9dc5; + for (let i = 0; i < id.length; i++) { + hash ^= id.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash % shardCount; +} + +/** + * Strategy 1 (baseline) - exactly what `StubOracle` does today: read both + * JSON files, parse them, and index scores by plain-object property access. + */ +const rawJson = { + id: 'raw-json', + label: 'Raw JSON + object lookup (baseline)', + build(corpusDir, artifactDir) { + freshDir(artifactDir); + cpSync(join(corpusDir, 'destinations.json'), join(artifactDir, 'destinations.json')); + cpSync(join(corpusDir, 'scores.json'), join(artifactDir, 'scores.json')); + return measureArtifact(artifactDir, ['destinations.json', 'scores.json']); + }, + async load(artifactDir) { + const destinations = JSON.parse( + readFileSync(join(artifactDir, 'destinations.json'), 'utf-8') + ).destinations; + const scores = JSON.parse(readFileSync(join(artifactDir, 'scores.json'), 'utf-8')); + return { + size: destinations.length, + retain: { destinations, scores }, + lookup(id) { + return scores[id]; + } + }; + } +}; + +/** + * Strategy 2 - same artifact, but parsed once into a `Map`. Costs a little + * more at startup and buys prototype-safe, monomorphic lookups. + */ +const rawJsonIndex = { + id: 'raw-json-index', + label: 'Raw JSON + Map index', + build(corpusDir, artifactDir) { + return rawJson.build(corpusDir, artifactDir); + }, + async load(artifactDir) { + const destinations = JSON.parse( + readFileSync(join(artifactDir, 'destinations.json'), 'utf-8') + ).destinations; + const scores = JSON.parse(readFileSync(join(artifactDir, 'scores.json'), 'utf-8')); + const index = new Map(); + for (const d of destinations) { + index.set(d.id, { score: scores[d.id], label: d.label, risk_pattern: d.risk_pattern }); + } + return { + size: index.size, + retain: index, + lookup(id) { + return index.get(id)?.score; + } + }; + } +}; + +/** + * Strategy 3 - ship a generated ES module instead of data files. The engine + * parses JS rather than JSON, but the artifact is importable from a bundler + * with no filesystem or fetch access at all. + */ +const esmModule = { + id: 'esm-module', + label: 'Generated ES module', + build(corpusDir, artifactDir) { + freshDir(artifactDir); + const { destinations, scores } = readCorpus(corpusDir); + const rows = destinations.map((d) => [d.id, scores[d.id], d.label, d.risk_pattern]); + const source = + '// Generated by scripts/bench/strategies.mjs - do not edit.\n' + + `export const rows = ${JSON.stringify(rows)};\n` + + 'export const index = new Map(rows.map((r) => [r[0], r[1]]));\n' + + 'export function getScore(id) {\n return index.get(id);\n}\n'; + writeFileSync(join(artifactDir, 'fixtures.mjs'), source); + return measureArtifact(artifactDir, ['fixtures.mjs']); + }, + async load(artifactDir) { + // Cache-bust so repeated loads in one process are not free. + const url = `${pathToFileURL(join(artifactDir, 'fixtures.mjs')).href}?v=${Math.random()}`; + const mod = await import(url); + return { + size: mod.index.size, + retain: mod, + lookup(id) { + return mod.getScore(id); + } + }; + } +}; + +/** + * Strategy 4 - one gzipped JSON blob, inflated at startup. Smallest thing to + * ship and to host; pays decompression on every cold start. + */ +const gzipJson = { + id: 'gzip-json', + label: 'Gzipped JSON archive', + build(corpusDir, artifactDir) { + freshDir(artifactDir); + const { destinations, scores } = readCorpus(corpusDir); + const rows = destinations.map((d) => [d.id, scores[d.id], d.label, d.risk_pattern]); + writeFileSync(join(artifactDir, 'fixtures.json.gz'), gzipSync(Buffer.from(JSON.stringify(rows)))); + return measureArtifact(artifactDir, ['fixtures.json.gz']); + }, + async load(artifactDir) { + const { gunzipSync } = await import('node:zlib'); + const raw = gunzipSync(readFileSync(join(artifactDir, 'fixtures.json.gz'))).toString('utf-8'); + const index = new Map(JSON.parse(raw).map((r) => [r[0], r[1]])); + return { + size: index.size, + retain: index, + lookup(id) { + return index.get(id); + } + }; + } +}; + +/** + * Strategy 5 - split packs. Startup reads only a tiny manifest; each shard is + * pulled in the first time a lookup lands in it. Shard placement is a hash of + * the ID, so no global index has to be shipped or held in memory. + */ +const splitPacks = { + id: 'split-packs', + label: 'Split packs, lazily loaded', + build(corpusDir, artifactDir) { + freshDir(artifactDir); + const { destinations, scores } = readCorpus(corpusDir); + const shardCount = Math.max(1, Math.ceil(destinations.length / SHARD_SIZE)); + const shards = Array.from({ length: shardCount }, () => ({})); + + for (const d of destinations) { + shards[shardOf(d.id, shardCount)][d.id] = scores[d.id]; + } + + mkdirSync(join(artifactDir, 'packs'), { recursive: true }); + shards.forEach((shard, i) => { + writeFileSync(join(artifactDir, 'packs', `${i}.json`), JSON.stringify(shard)); + }); + writeFileSync( + join(artifactDir, 'manifest.json'), + JSON.stringify({ + version: 1, + hash: 'fnv1a', + shardCount, + total: destinations.length, + packs: shards.map((s, i) => ({ file: `packs/${i}.json`, entries: Object.keys(s).length })) + }) + ); + return measureArtifact(artifactDir, ['manifest.json']); + }, + async load(artifactDir) { + const manifest = JSON.parse(readFileSync(join(artifactDir, 'manifest.json'), 'utf-8')); + const loaded = new Map(); + let touchedBytes = 0; + + function shard(n) { + let entries = loaded.get(n); + if (!entries) { + const buf = readFileSync(join(artifactDir, manifest.packs[n].file)); + touchedBytes += buf.length; + entries = new Map(Object.entries(JSON.parse(buf.toString('utf-8')))); + loaded.set(n, entries); + } + return entries; + } + + return { + size: manifest.total, + retain: { manifest, loaded }, + lookup(id) { + return shard(shardOf(id, manifest.shardCount)).get(id); + }, + touchedBytes() { + return touchedBytes; + }, + loadedShards() { + return loaded.size; + } + }; + } +}; + +export const STRATEGIES = [rawJson, rawJsonIndex, esmModule, gzipJson, splitPacks]; + +export function getStrategy(id) { + const strategy = STRATEGIES.find((s) => s.id === id); + if (!strategy) { + throw new Error(`unknown strategy "${id}" (have: ${STRATEGIES.map((s) => s.id).join(', ')})`); + } + return strategy; +} diff --git a/scripts/bench/strkey.mjs b/scripts/bench/strkey.mjs new file mode 100644 index 0000000..651c305 --- /dev/null +++ b/scripts/bench/strkey.mjs @@ -0,0 +1,85 @@ +/** + * Minimal Stellar strkey encoder, used only to mint *synthetic* account + * public keys for benchmark corpora. + * + * Only the `G...` (ed25519 public key) form is implemented on purpose: the + * benchmark corpora must never contain anything that could be mistaken for a + * secret seed, and `scripts/check-secrets.mjs` guards the repository against + * exactly that. + */ + +const BASE32_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; + +/** Version byte for an ed25519 public key: 6 << 3. */ +const VERSION_BYTE_ACCOUNT_ID = 6 << 3; + +function crc16xmodem(bytes) { + let crc = 0; + for (const byte of bytes) { + crc ^= byte << 8; + for (let i = 0; i < 8; i++) { + crc = crc & 0x8000 ? ((crc << 1) ^ 0x1021) & 0xffff : (crc << 1) & 0xffff; + } + } + return crc; +} + +function base32Encode(bytes) { + let out = ''; + let bits = 0; + let value = 0; + for (const byte of bytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + out += BASE32_ALPHABET[(value >>> bits) & 0x1f]; + } + } + if (bits > 0) { + out += BASE32_ALPHABET[(value << (5 - bits)) & 0x1f]; + } + return out; +} + +/** + * Encode 32 raw bytes as a Stellar account public key (`G...`, 56 chars). + * + * @param {Uint8Array} payload 32-byte ed25519 public key material + * @returns {string} + */ +export function encodeAccountId(payload) { + if (payload.length !== 32) { + throw new Error(`expected 32-byte payload, got ${payload.length}`); + } + const body = new Uint8Array(1 + 32); + body[0] = VERSION_BYTE_ACCOUNT_ID; + body.set(payload, 1); + + const checksum = crc16xmodem(body); + const full = new Uint8Array(body.length + 2); + full.set(body, 0); + // CRC16 is appended little-endian. + full[body.length] = checksum & 0xff; + full[body.length + 1] = (checksum >>> 8) & 0xff; + + return base32Encode(full); +} + +/** + * Deterministic 32-bit PRNG (mulberry32). Seeded runs make every corpus in + * this harness byte-for-byte reproducible across machines and CI. + * + * @param {number} seed + * @returns {() => number} generator returning floats in [0, 1) + */ +export function seededRandom(seed) { + let state = seed >>> 0; + return function next() { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/scripts/check-secrets.mjs b/scripts/check-secrets.mjs index 9b247e4..343e694 100644 --- a/scripts/check-secrets.mjs +++ b/scripts/check-secrets.mjs @@ -8,6 +8,12 @@ const root = fileURLToPath(new URL('..', import.meta.url)); // Stellar secret seed: starts with S, 55 alphanumeric chars, valid Ed25519 checksum const SECRET_SEED_REGEX = /\bS[A-Z2-7]{55}\b/g; +// A scanner that silently stops matching is worse than no scanner at all, so +// prove the validator still recognises a well-formed secret seed before a clean +// run is allowed to mean anything. The seed is derived at runtime from a fixed +// payload - no secret material is stored in this file. +selfTest(); + // Get all tracked files const output = execSync('git ls-files', { encoding: 'utf-8', cwd: root }); const files = output.trim().split('\n').filter(Boolean); @@ -23,7 +29,7 @@ for (const file of files) { // Validate Ed25519 secret seed checksum if (isValidEd25519SecretSeed(seed)) { const line = content.substring(0, match.index).split('\n').length; - console.error(\SECRET SEED FOUND: \:\ — [REDACTED]\); + console.error(`SECRET SEED FOUND: ${file}:${line} - [REDACTED]`); found = true; } } @@ -37,9 +43,9 @@ if (found) { process.exit(1); } -console.log('No Stellar secret seeds found — CI check passed.'); +console.log('No Stellar secret seeds found - CI check passed.'); -// Validate Ed25519 secret seed checksum (last byte is CRC16-XModem of first 55 bytes) +// Validate Ed25519 secret seed checksum (trailing 2 bytes of the decoded payload) function isValidEd25519SecretSeed(seed) { try { // Decode base32 @@ -58,9 +64,11 @@ function isValidEd25519SecretSeed(seed) { } } if (bytes.length < 3) return false; - // Last 2 bytes are CRC16-XModem checksum + // The trailing 2 bytes are a CRC16-XModem of everything before them, + // appended little-endian (SEP-23). Reading them big-endian makes this + // function reject every real strkey. const dataBytes = bytes.slice(0, -2); - const expected = (bytes[bytes.length - 2] << 8) | bytes[bytes.length - 1]; + const expected = bytes[bytes.length - 2] | (bytes[bytes.length - 1] << 8); const actual = crc16xmodem(dataBytes); return expected === actual; } catch { @@ -78,3 +86,35 @@ function crc16xmodem(data) { } return crc; } + +function selfTest() { + const body = [18 << 3, ...new Array(32).fill(7)]; // version byte for an ed25519 secret seed + const crc = crc16xmodem(body); + const knownGood = base32Encode([...body, crc & 0xFF, (crc >>> 8) & 0xFF]); + + const matches = SECRET_SEED_REGEX.test(knownGood); + SECRET_SEED_REGEX.lastIndex = 0; // .test() on a /g regex advances lastIndex + + if (!matches || !isValidEd25519SecretSeed(knownGood)) { + console.error('check-secrets self-test failed: this scanner no longer recognises a valid'); + console.error('Stellar secret seed, so a clean result would be meaningless. Fix the detector.'); + process.exit(1); + } +} + +function base32Encode(bytes) { + const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; + let out = ''; + let bits = 0; + let value = 0; + for (const byte of bytes) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + bits -= 5; + out += ALPHABET[(value >>> bits) & 0x1F]; + } + } + if (bits > 0) out += ALPHABET[(value << (5 - bits)) & 0x1F]; + return out; +}