diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3fc2e2c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,86 @@ +# The whole test suite is `node --test` over plain modules, so CI needs no +# services, no secrets and no network. Anything here that WOULD need an API key +# is deliberately left out: a green tick that depends on a repository secret is +# a tick contributors cannot reproduce, and forks cannot run at all. +name: CI + +on: + push: + branches: [main, dev] + pull_request: + branches: [main, dev] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # 20 is the floor declared in engines; 22 is the current LTS. + node: [20, 22] + name: test (node ${{ matrix.node }}) + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + + - name: Install + run: npm ci --no-audit --no-fund + + - run: npm test + + smoke: + runs-on: ubuntu-latest + name: cli and rubric packs + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - name: Install + run: npm ci --no-audit --no-fund + + # No arguments prints usage and must exit 0. `doctor` is NOT run here: + # it requires a provider key and exits 1 without one, so in CI it would + # only ever prove that the secret is missing. + - name: CLI help path exits 0 + run: node src/cli.mjs + + # validatePack is called directly so the packs are checked for structural + # errors without a key, a network call, or the rest of doctor. + - name: Validate every rubric pack + run: | + node -e ' + const { readdirSync } = require("node:fs"); + (async () => { + const dir = "./src/rubrics"; + let checked = 0, bad = 0; + const { validatePack } = await import(dir + "/pack.mjs"); + for (const file of readdirSync(dir).filter((f) => f.endsWith(".mjs") && f !== "pack.mjs")) { + const mod = await import(dir + "/" + file); + for (const [name, value] of Object.entries(mod)) { + if (!value || typeof value !== "object" || !value.questions) continue; + checked++; + const problems = validatePack(value); + if (problems.length) { + bad++; + console.error("FAIL " + file + " -> " + name); + for (const p of problems) console.error(" " + p); + } else { + console.log("OK " + file + " -> " + name + " (" + Object.keys(value.questions).length + " questions)"); + } + } + } + if (!checked) { console.error("no rubric packs found"); process.exit(1); } + process.exit(bad ? 1 : 0); + })(); + ' diff --git a/.gitignore b/.gitignore index 163e13d..3c89ef4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,17 @@ node_modules/ .env .env.* !.env.example -# Never commit anyone's exported content or derived ratings. + +# Never commit anyone's exported content or anything derived from it. +# report.json and dashboard.html both embed the full text of every post, so the +# output directory is ignored wholesale rather than by file type. An earlier +# version listed `*.jsonl` and `dashboard.html`, which missed `jev-out/report.json` +# entirely. +jev-out/ data/ out/ *.jsonl dashboard.html +report.json + .DS_Store diff --git a/AGENTS.md b/AGENTS.md index 3386d01..d40c787 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,27 +34,41 @@ If a change makes the output more impressive and less defensible, it is wrong. ``` src/stats.mjs rank correlation, permutation tests, the power gate src/report.mjs the statistics -> English boundary. Read this first. +src/dashboard.mjs buildReport output -> one self-contained HTML string +src/index.mjs library entry point, re-exporting the stable surface src/rubrics/pack.mjs pack format and validation -src/rubrics/*.mjs shipped packs +src/rubrics/*.mjs shipped packs, each with its own `directions` export src/adapters/*.mjs platform export parsing src/providers/*.mjs one file per transport -src/cli.mjs doctor / build / rate / analyze / run -skills/jev-writer/ the agent skill and its references +src/cli.mjs doctor / build / rate / analyze / dashboard / run +skills/jev-writer/ the agent skill and its five reference files test/ node:test, no framework +.github/workflows/ci.yml node 20 and 22, plus a keyless pack-validation smoke job +llms.txt the short machine-readable description of the project ``` `src/report.mjs` is the most important file to understand. Everything above it speaks statistics; everything below it speaks English. No p-value, rho or confidence interval may cross that boundary onto a user-facing surface. +`src/dashboard.mjs` does no I/O. It takes what `buildReport` returned and gives +back a string; the CLI owns reading and writing files. Keep it that way, so the +page can be rendered from a test or an agent without touching a disk. + ## Conventions - **ESM only**, Node >= 20, no TypeScript, no build step. - **One runtime dependency** (`ai`, required by the AI Gateway path). Do not add more without a strong reason; the CSV and xlsx readers are hand-written specifically to avoid dependencies. +- **The `exports` map in `package.json` is the public surface.** Consumers import + `jev-writer`, `jev-writer/stats.mjs`, `jev-writer/report.mjs`, + `jev-writer/dashboard.mjs`, `jev-writer/rubrics/pack.mjs`, + `jev-writer/rubrics/linkedin-post.mjs` or `jev-writer/adapters/linkedin.mjs`. + A deep path into `src/` no longer resolves from outside the package, so adding + a module that users need means adding a subpath to that map as well. - **Comments explain why, not what.** Several comments in this repo record a - measured fact or a bug that was actually hit — those are load-bearing + measured fact or a bug that was actually hit. Those are load-bearing documentation, not clutter. Do not strip them. - Tests use `node:test` and synthetic fixtures. **No real post data in the repository, ever.** @@ -71,13 +85,51 @@ confidence interval may cross that boundary onto a user-facing surface. SDK rejects with "did not select a highest-probability option". That is sampling noise on a genuinely ambiguous item; retry the repetition, never drop the item. -- **`score` accepts 2–10 levels only.** The API rejects anything else. +- **`score` accepts 2 to 10 levels only.** The API rejects anything else. +- **`validatePack` does not check directions.** It checks ids, types, tiers, + instructions, score-level counts, the Choice no-match option, and that every + backticked reference in an instruction names a declared `stateField`. Direction + of merit is a separate `directions` export beside the pack, and it is the + report layer that enforces it: `buildReport` throws when a rated dimension has + no direction, and `cli.mjs` skips report generation with a warning when a pack + has no directions entry at all. A pack can therefore be structurally valid and + still produce no `report.json`. - **Rubric wording is load-bearing.** Tightening one clause moved a judgment from 0.99 to 0.71 on identical input. Treat pack edits as breaking changes: bump the version and re-rate rather than pooling ratings across wordings. - **Bands are direction-aware.** Never colour or rank by a raw rubric value. - **Bands are quintiles**, so "share of posts in the bottom two bands" is 40% by construction. That bug shipped once; do not reintroduce it. +- **A pack's `groups` are presentation only.** They give the dashboard families + for the expanded post view and never touch how a question is asked, rated or + correlated. A pack that declares none still renders, with everything falling + into one catch-all list. +- **The LinkedIn adapter reports rather than guesses.** Two posts on the same + date cannot be told apart, because the analytics export joins on date, so + metrics, post URL and media fields are all left null on those days. The + follower count is exempt on purpose: it is a per-day value, so two posts on one + date genuinely share it. Unparseable dates, and a follower total that fails its + consistency check against the daily rows, each raise a warning rather than + failing silently. Surface `warnings` to the user; do not swallow them. + +### Two statistical bugs that were live, and must not come back + +- **Slicing the control array.** An earlier version built the publication-date + control over the whole subset and then took `control.slice(0, pairs.length)`. + That is the FIRST n dates, not the dates OF the n surviving items, so a single + post dropped for a missing rating shifted every later date onto the wrong post + and silently corrupted the date control that decides whether a finding is + confirmed. It is fixed in both `src/report.mjs` and `src/cli.mjs` by carrying + `[value, outcome, date]` triples through one filter, which makes the + misalignment unrepresentable. Never reintroduce a parallel array that has to be + filtered in step with another. +- **Truthiness on a metric.** `impressions && engagements ? ... : null` treated a + genuine zero as missing data. Zero engagements on a real post is the single + most informative outcome in the file, and dropping it is not random: it removes + the worst-performing posts from every correlation and pulls the whole analysis + toward "everything I write is fine". Only a zero *denominator* is undefined. + Test `Number.isFinite`, never truthiness, on anything that can legitimately be + zero. ## Statistics rules @@ -86,26 +138,54 @@ confidence interval may cross that boundary onto a user-facing surface. between levels. - **Permutation p-values and bootstrap CIs**, not normal approximations; n is routinely under 60. -- **Control for publication date** on any headline result. A rated dimension and - an outcome that both drift over time correlate for no reason. +- **Resampling is seeded, and the seed is reported.** `permutationP`, + `bootstrapCI` and `testDimension` all take an overridable `seed`, defaulting to + the exported `DEFAULT_SEED` (20260920), and `testDimension` puts the seed it + used on the row it returns. A published finding is therefore reproducible by + anyone holding the same data, and a finding that survives on only one seed was + never a finding. +- **Control for publication date** on any headline result, and run the + false-discovery correction on the controlled p-value rather than the raw one. + A rated dimension and an outcome that both drift over time correlate for no + reason, and correcting the uncontrolled p would let a result the tool itself + flagged as shared time drift reach the headline. - **Keep the two outcomes separate.** `reach_rate` is impressions over followers at post time; `conversion_rate` is engagements over impressions. They answer different questions and must not be blended into one score. ## Privacy -This tool sends private writing to a third party. `.gitignore` excludes `data/`, -`out/` and `*.jsonl`. A generated dashboard embeds full post text, so it is -generated, never committed. Zero Data Retention is requested on every gateway -call. Never ask a user to paste an API key into a chat. +This tool sends private writing to a third party. `.gitignore` excludes +`jev-out/`, `data/`, `out/`, `*.jsonl`, `dashboard.html` and `report.json`. Both +`report.json` and the generated dashboard embed the full text of every post, so +they are generated, never committed. Zero Data Retention is requested on every +gateway call. Never ask a user to paste an API key into a chat. + +## The skill + +The agent skill lives in `skills/jev-writer/` and is installed with: + +```bash +npx skills add Kaos599/jev-writer --skill jev-writer +``` + +The full URL form works too and installs the same skill. If you change the skill, +change `SKILL.md` and its reference files together: anything too long for the +skill body belongs in `skills/jev-writer/references/`, and `SKILL.md` should +point at it rather than repeat it. ## Before you open a PR ```bash -npm test # node:test, all must pass +npm test # node:test, 67 tests, all must pass node src/cli.mjs doctor # validates every pack and makes one live call ``` +CI runs the suite on Node 20 and 22 and separately validates every rubric pack +without a key. `doctor` is deliberately not run in CI: it needs a provider key +and exits 1 without one, so a green tick there would only ever prove that a +repository secret exists. Run it locally before you open the PR. + Add a test for any new statistic or pack-validation rule. Several existing tests -exist because they caught a real bug — read them before changing behaviour they +exist because they caught a real bug; read them before changing behaviour they pin. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dfb919e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,39 @@ +# Changelog + +Notable changes to jev-writer. Format loosely follows [Keep a Changelog](https://keepachangelog.com/). + +## [Unreleased] + +### Added +- `jev-writer dashboard` renders `report.json` as one self-contained HTML page: inline CSS and JS, data embedded, no CDN and no build step, so it opens over `file://`. Exposed as `buildDashboard()` for programmatic use. +- `corpus` facts on the report: posting volume by year, follower trajectory, and headline medians. +- A derived posting-cadence classification, so the dashboard writes a true sentence about your volume instead of a templated one. +- An `exports` map, making `jev-writer/rubrics/pack.mjs`, `jev-writer/stats.mjs`, `jev-writer/report.mjs`, `jev-writer/dashboard.mjs` and `jev-writer/adapters/linkedin.mjs` the stable public surface. +- Question groups on the LinkedIn pack, covering all 30 questions across 5 families. Presentation only. +- CI on Node 20 and 22, plus a smoke job validating every rubric pack without a key. +- Tests for `report.mjs` and `dashboard.mjs`, which had none. Suite grew from 27 to 67. +- `CONTRIBUTING.md`, `SECURITY.md`, and this file. + +### Fixed +- **The dashboard reported "0.0% reach" when there was no engagement data at all.** `null * 100` is `0` in JavaScript, so the percentage formatter turned "we cannot measure this" into a confident claim of zero reach, on the hero card, whenever no analytics export was supplied. It now reads "No data". +- Benjamini-Hochberg corrected over the dimensions that could actually be tested while the power headline announced dimensions times outcomes. Where outcome coverage is thin the two diverge. The family size is now computed once and the number announced is the number corrected over. +- `cadenceOf` picked the peak year with a strict `>`, so an author whose latest year tied their best year was told they were publishing less. +- `buildDashboard` threw an opaque `TypeError` on a `report.json` written before the corpus fields existed. It now names the stale file and the command to re-run. +- **The date-control gate was silently corrupted.** `control.slice(0, pairs.length)` took the first n publication dates rather than the dates of the n surviving items, so a single post missing a rating misaligned every date after it. This is the mechanism that decides whether a finding is confirmed or dismissed as time drift. Values, outcomes and dates now travel as one tuple through a single filter, making the misalignment unrepresentable. +- **Zero engagements were treated as missing data.** `impressions && engagements ? ... : null` dropped any post with no engagement from every correlation, which systematically excluded the worst-performing posts and biased every finding upward. Only a zero denominator now yields null. +- Same-date collisions left media type and media count attributed to a pooled guess while correctly nulling metrics. All per-post fields are now null on a colliding date. +- Rows with unparseable dates were dropped in four places without a word. They are now counted and reported, with sample values, in `warnings`. +- The follower back-walk assumed the export's headline total aligned with the last daily row, unvalidated. A mismatch or a moved sheet layout now nulls follower-derived fields rather than biasing every reach rate. +- Posts published before the follower window received the earliest known follower count. They now receive none. +- The permutation seed is overridable and reported, rather than a hardcoded default no caller could reach. +- `.gitignore` missed `jev-out/report.json`, the one output containing every post in full. +- The lockfile still carried the pre-rename package name. + +### Changed +- The dashboard's narrative is derived from the data rather than hardcoded. An earlier version's heading asserted that the author had stopped posting, which was true of exactly one corpus. +- Methods text derives its question counts from the pack instead of hardcoding them. +- Platform names are parameterised rather than assuming LinkedIn. + +## [0.1.0] + +Initial public release: providers, statistics, rubric packs, the LinkedIn adapter, the report layer, and the agent skill. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d571084 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,52 @@ +# Contributing + +New rubric packs and platform adapters are the most useful contributions. Everything here is plain ES modules on Node 20 or newer, with one runtime dependency. + +## Setup + +```bash +git clone https://github.com/Kaos599/jev-writer && cd jev-writer +npm install +npm test # 67 tests, no network, no key required +node src/cli.mjs doctor # validates every pack and makes one live call +``` + +`doctor` is the only command that needs a provider key. CI never runs it, because a green tick that depends on a repository secret is one contributors cannot reproduce and forks cannot run at all. + +## What makes a good contribution here + +This project's whole claim is that it refuses to flatter its user. Contributions are judged against that first. + +**A new rubric pack** must tag every question `primary` or `exploratory` before any result is seen, declare a direction of merit for each dimension, and pass `validatePack`. Ask the model nothing that code can count: length, emoji, hashtags and cadence are computed exactly and compete with the judgments on equal footing. + +**A new platform adapter** must produce items carrying text, a date, and at least one outcome. Where a value cannot be attributed to a specific item, leave it null. Do not guess and do not carry a neighbour's value forward. If rows are dropped, count them and say so in `warnings`; a silent drop is worse than a loud failure because it looks like a small corpus rather than a broken parser. + +**A statistics change** needs a test that fails without it. The module is small on purpose. + +## Testing + +```bash +npm test # everything +node --test test/report.test.mjs # one file +``` + +Tests use `node:test` and `node:assert/strict` with no assertion library and no mocking framework. The adapter tests write real ZIP and XLSX files to temporary directories rather than mocking the readers, because the defects those tests exist to catch are hardcoded sheet offsets that a mock would happily preserve. + +## Traps worth knowing before you start + +These have each cost someone a day. [AGENTS.md](AGENTS.md) carries the full list. + +- Bands are quintiles of the author's own distribution, so "share of posts in the bottom two bands" is 40% for every dimension by construction. It is not a measure of weakness. +- Rubric wording is load-bearing. Tightening one clause moved a probability from 0.99 to 0.71 on identical input. Treat a pack edit as a breaking change: bump the version and re-rate rather than pooling ratings across wordings. +- Score magnitudes are ordinal, not numeric. Every correlation is Spearman rank. Never Pearson. +- Parallel arrays built by two independent filters will drift apart. Carry the tuple through one filter instead. + +## Pull requests + +Branch from `dev`, not `main`. Keep commits logical rather than chronological. CI runs the suite on Node 20 and 22 plus a pack-validation smoke job; both must pass. + +Commit messages use `type(scope): what changed`, written so someone reading `git log` a year from now learns something. Explain why in the body when the why is not obvious. + +## Reporting a problem + +Wrong numbers matter more than crashes here. If a finding looks wrong, open an issue with the shape of your corpus (post count, how many carry outcome data, date range) and the relevant part of `report.json`. Never attach `report.json` or `dashboard.html` themselves: both embed the full text of every post you have written. diff --git a/README.md b/README.md index c7f8a65..705f324 100644 --- a/README.md +++ b/README.md @@ -4,21 +4,22 @@ **Find out which qualities of your writing actually predict engagement.** +[![CI](https://github.com/Kaos599/jev-writer/actions/workflows/ci.yml/badge.svg)](https://github.com/Kaos599/jev-writer/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Node](https://img.shields.io/badge/node-%3E%3D20-brightgreen.svg)](package.json) -[![Tests](https://img.shields.io/badge/tests-27%20passing-brightgreen.svg)](test/) +[![Tests](https://img.shields.io/badge/tests-67%20passing-brightgreen.svg)](test/) --- -jev-writer rates every post you have published against a pre-registered rubric, then tests those ratings against your real engagement data. It reports what it finds in plain language, and it withholds conclusions when your sample is too small to support them. +jev-writer rates every post you have published against a pre-registered rubric, then tests those ratings against your real engagement numbers. It reports what it finds in plain language, and it withholds conclusions when your sample is too small to support them. It analyses posts. It does not write them. ```bash -npx jev-writer doctor # check your key and packs, make one live call -npx jev-writer run ./exports # build corpus, rate it, analyse it +npx github:Kaos599/jev-writer doctor # check your key, make one live call +npx github:Kaos599/jev-writer run ./exports # build, rate, analyse, render ``` Works with LinkedIn exports today, and with any platform you write a rubric pack for. Bring a Vercel AI Gateway, OpenRouter, or TypeSafe key. @@ -27,13 +28,17 @@ Works with LinkedIn exports today, and with any platform you write a rubric pack ## Contents +- [What you can do with it](#what-you-can-do-with-it) +- [Features](#features) +- [Getting started](#getting-started) - [What problem this solves](#what-problem-this-solves) - [How this differs from asking an LLM to rate your posts](#how-this-differs-from-asking-an-llm-to-rate-your-posts) - [How it avoids telling you what you want to hear](#how-it-avoids-telling-you-what-you-want-to-hear) - [Which outcome it optimises for](#which-outcome-it-optimises-for) +- [The dashboard](#the-dashboard) +- [The writing prompt](#the-writing-prompt) - [Providers and keys](#providers-and-keys) - [What it costs to run](#what-it-costs-to-run) -- [Getting started](#getting-started) - [Writing your own rubric pack](#writing-your-own-rubric-pack) - [Using it from an agent](#using-it-from-an-agent) - [Known limits](#known-limits) @@ -41,6 +46,113 @@ Works with LinkedIn exports today, and with any platform you write a rubric pack --- +## What you can do with it + +Six things people actually use this for. Each one names the command that does it. + +**Find out which of your own writing habits predicted engagement.** The core loop. Thirty judgments per post, correlated against your real numbers, with everything that fails correction thrown away. `jev-writer run ./exports` + +**Get a review prompt built from your own results.** The tool writes a prompt naming the dimensions that predicted engagement *for you* and the ones you are habitually weakest on. Paste it into any assistant before you publish. It is not "write a better hook"; it is your results. `jev-writer analyze`, then read `writingPrompt` in `jev-out/report.json`. + +**See where every post stands without reading a statistic.** One self-contained HTML page: each post graded, each judgment shown as a coloured band, sortable and searchable. `jev-writer dashboard` + +**Audit a rubric before you trust it.** Packs are plain files. Change a level description, re-run, and see whether the finding survives. A full pass on 178 posts cost about $0.31, which is what makes this practical rather than theoretical. + +**Check whether an AI-judge pipeline you already run is measuring anything.** The statistics module stands alone. Feed it your own judge scores and your own outcomes and it will tell you how much survives date control and false-discovery correction. `import { testDimension } from 'jev-writer/stats.mjs'` + +**Score a platform this repo has never seen.** Write an adapter that emits items with text and an outcome, write a pack, and the entire pipeline applies. Newsletters, video titles, docs pages, cold email. + +Who this is for: people publishing regularly enough to have 25 or more posts with engagement data, who want to know whether their instincts about their own writing are real. Below that threshold the tool will tell you it cannot know, which is the point. + +## Features + +| | | +|---|---| +| **Calibrated judgments** | Ratings come from [Jev](https://typesafe.ai), a System One model returning probability distributions over levels you define, not generated text parsed into a number. | +| **Pre-registration** | Every question is tagged `primary` or `exploratory` in a file that lives in git, so the commit timestamp proves the commitment preceded the result. | +| **Power gate** | Under 25 posts with outcome data, correlations are refused outright. Between 25 and 60 the tool reports the smallest effect your sample can detect. | +| **False-discovery correction** | Benjamini-Hochberg across every test run, not just the ones that looked good. The family size is the number of tests actually performed, and the same number is what the tool reports to you. | +| **Confound control** | Every result is re-tested as a partial correlation controlling for publication date, and the correction runs on the controlled p-value. | +| **Direction of merit** | Each dimension declares `higher_is_better`, `lower_is_better`, or `neutral`, so a low score where low is good paints green. 13, 7 and 4 of them respectively in the shipped pack. | +| **Code features compete** | Character count, emoji, hashtags, sentence length and posting cadence are computed exactly and ranked against the model's judgments in the `analyze` table. If raw length beats your rubric, you find out. | +| **Self-contained dashboard** | One HTML file, no CDN, no build step, no network. Opens over `file://`. | +| **Generated writing prompt** | A paste-ready review prompt derived from your own validated findings. | +| **Portable rubric packs** | One pack, three provider dialects, translated at the boundary. | +| **Zero-dependency parsing** | CSV, ZIP and XLSX readers are written against the stdlib. The only runtime dependency is `ai`. | +| **Agent skill included** | A skill for Claude Code, Codex and similar, with five reference files. | +| **67 tests** | `node --test`, no assertion library, no network. | + +## Getting started + +### 1. Export your data + +On LinkedIn you need **both** of these. They are different exports and neither contains the other. + +| Export | Where | Contains | +|---|---|---| +| Data archive | Settings → Data Privacy → Get a copy of your data → **the larger archive** | post text, dates, URLs | +| Post analytics | Analytics & Tools → Post analytics → Export | impressions, reactions, comments, daily followers | + +The basic archive contains no `Shares_*.csv` at all, which is the single most common setup failure. Request the larger one first; it takes hours to arrive. Put both files, still zipped, in one directory. + +Other platforms are documented in [`references/platform-exports.md`](skills/jev-writer/references/platform-exports.md). + +### 2. Get a key + +Any one of three providers works. Full instructions per provider in [`references/providers.md`](skills/jev-writer/references/providers.md). + +```bash +export AI_GATEWAY_API_KEY=... # or OPENROUTER_API_KEY, or TYPESAFE_API_KEY +``` + +Set exactly one. The tool detects which and translates the rubric into that provider's dialect. + +### 3. Run it + +```bash +npx github:Kaos599/jev-writer doctor +npx github:Kaos599/jev-writer run ./exports +``` + +Or from a clone, which is what you want if you plan to edit a rubric: + +```bash +git clone https://github.com/Kaos599/jev-writer && cd jev-writer +npm install +node src/cli.mjs doctor +node src/cli.mjs run ./exports +``` + +`doctor` checks Node, your key, every rubric pack, and makes one live call before a full run spends anything. + +### 4. Read the output + +Everything lands in `jev-out/` (override with `JEV_WRITER_OUT`). + +| File | What it is | +|---|---| +| `corpus.jsonl` | your posts, joined to their metrics, with code features computed | +| `ratings.jsonl` | raw judgments, appended as they arrive so a run resumes after an interruption | +| `report.json` | grades, bands, plain-language findings, weak spots, and the writing prompt | +| `dashboard.html` | the whole report as one page | + +```bash +open jev-out/dashboard.html +``` + +`jev-out/` is gitignored. Both `report.json` and `dashboard.html` embed the full text of every post, so keep them out of commits. + +### Commands + +| Command | What it does | +|---|---| +| `doctor` | check Node, keys, packs; one live call | +| `build ` | parse exports into `corpus.jsonl` | +| `rate` | rate the corpus, resumable | +| `analyze` | correlate against outcomes, write `report.json` | +| `dashboard` | render `report.json` to `dashboard.html` | +| `run ` | build, rate, analyze, dashboard | + ## What problem this solves Most tools that rate your content ask a language model to score a post from 1 to 10, then present whatever comes back as insight. @@ -65,9 +177,17 @@ Four mechanisms, each of which can cost you a finding. **A power gate.** Below 25 posts with outcome data, the tool computes descriptive statistics and refuses to correlate at all. Between 25 and 60 it reports the minimum effect your sample can detect, and how many "significant" results to expect by chance. -**Confound control.** Every headline result is re-tested as a partial correlation controlling for publication date. A rated dimension and an outcome that both drift over time will correlate for no reason. +**Confound control.** Every headline result is re-tested as a partial correlation controlling for publication date. A rated dimension and an outcome that both drift over time will correlate for no reason. The false-discovery correction then runs on the controlled p-value, not the raw one. + +**Code features compete.** Character count, emoji, hashtags, sentence length, and posting cadence are computed exactly and ranked alongside the model's judgments, so a rubric that is merely restating post length is visible as such. + +One scope note, because it matters: those code features appear in the `analyze` terminal table, which is where you compare them against your rubric. `report.json` and the dashboard cover the rubric dimensions only. The two therefore correct over different families and can disagree about which dimensions clear the threshold. Read the table when you are auditing a rubric, and the dashboard when you are reading results. + +What this looks like in practice, on this project's own corpus. 178 posts, 43 of them carrying reach data, 41 carrying conversion data. The shipped pack rates 24 dimensions, so 24 tests per outcome and 48 in total, of which about 1.2 per outcome are expected to clear p < 0.05 on noise alone. -**Code features compete.** Character count, emoji, hashtags, and posting cadence are computed exactly and ranked alongside the model's judgments. If raw length beats your rubric, you find out. +Ten relationships cleared it. **One** survived Benjamini-Hochberg, and that one was tagged exploratory, so the tool declined to promote it to a finding and reported a confirmed-findings count of zero. + +Correction runs within each outcome, on the date-controlled p-value rather than the raw one, and the number of tests the tool announces is the number it actually corrected over. A dimension with too little paired data to test is not counted as a test that the correction survived. ## Which outcome it optimises for @@ -80,7 +200,25 @@ Two, kept deliberately separate, because they answer different questions. Conversion rate is the cleaner test of craft. It is unconfounded by follower count, posting time, and algorithmic luck. Most analytics tools report raw reactions, which mostly measures reach. A post seen by 800 people that earned 60 engagements tells you more about your writing than one seen by 5,000 that earned 90. -Follower count at post time is reconstructed from the daily follower sheet in LinkedIn's own analytics export, so reach rate is a real ratio rather than a raw count compared across years of audience growth. +Follower count at post time is reconstructed from the daily follower sheet in LinkedIn's own analytics export, so reach rate is a real ratio rather than a raw count compared across years of audience growth. The reconstruction is validated against the export's headline total; if they disagree, follower-derived fields are nulled rather than guessed. + +## The dashboard + +```bash +jev-writer dashboard && open jev-out/dashboard.html +``` + +One HTML file. Inline CSS, inline JS, data embedded, no CDN and no fetch, so it opens on a laptop with no network. A 178-post corpus renders to about 600 KB. + +The presentation rules are the reason it exists. An earlier version showed rank correlations and p-values and was correctly described by its first reader as unreadable. So: no statistics vocabulary anywhere except one collapsed methods section, every judgment shown as a coloured band rather than a number, bands arriving direction-aware from the report so a low score on a dimension where low is good still paints green, and charts only where a chart genuinely says something. + +Every narrative sentence is derived from your data rather than templated. If your posting volume grew, it says so; the heading is not fixed in advance. + +## The writing prompt + +`report.json` carries a `writingPrompt` built from your own results: the dimensions that predicted engagement for you, what each one means, the level to aim for, and the gap between your typical post and your best ones. Paste it above a draft in any assistant. + +It is `null` when nothing survived validation. A prompt generated from no findings would be generic advice wearing your data as a costume. ## Providers and keys @@ -102,31 +240,6 @@ That matters for method rather than budget. Because a full pass is nearly free, > Check current pricing before a large run. TypeSafe launched in September 2026 and pricing is still moving. -## Getting started - -**1. Export your data.** On LinkedIn you need both of these: - -| Export | Path | Contains | -|---|---|---| -| Data archive | Settings → Data Privacy → Get a copy of your data → **larger archive** | post text, dates, URLs | -| Post analytics | Creator Mode → Analytics & Tools → Post analytics → Export | impressions, reactions, comments | - -The basic archive contains no `Shares_*.csv` at all, which is the single most common setup failure. Request the larger one first, since it takes hours. Put both files in one directory. - -**2. Get a key.** Any provider from the table above. - -**3. Run it.** - -```bash -export AI_GATEWAY_API_KEY=... -npx jev-writer doctor -npx jev-writer run ./exports -``` - -`doctor` verifies Node, your key, the pack, and makes one live call before a full run spends anything. - -Output lands in `jev-writer-out/`. The file you want is `report.json`: grades, bands, plain-language findings, and a writing prompt generated from your own results. - ## Writing your own rubric pack A pack is a versioned set of typed questions plus the fields each question may see. LinkedIn is one pack. Newsletters, video titles, documentation, and cold email are ordinary user-space files. @@ -163,14 +276,31 @@ Two rules when authoring: **Rubric wording is load-bearing.** During development, tightening one clause in a single question moved its probability from **0.99 to 0.71 on identical input**. `jev-1.13` reads instructions literally, as documented on TypeSafe's [model jaggedness page](https://docs.typesafe.ai/model-jaggedness/jev-1.13). Treat pack edits as breaking changes: bump the version and re-rate rather than pooling ratings across wordings. +Full guidance in [`references/rubric-authoring.md`](skills/jev-writer/references/rubric-authoring.md). + ## Using it from an agent -The repo ships an agent skill for Claude Code, Codex, and other agent environments. It walks a user through exporting their data, obtaining a key, running the pipeline, building a dashboard, and reading the result without overstating it. +The repo ships an agent skill for Claude Code, Codex, and other agent environments. It walks a user through exporting their data, obtaining a key, running the pipeline, opening the dashboard, and reading the result without overstating it. + +```bash +npx skills add Kaos599/jev-writer --skill jev-writer +``` + +The full URL form works too, and installs the same skill: ```bash npx skills add https://github.com/Kaos599/jev-writer --skill jev-writer ``` +Or install it by hand, which is just a file copy: + +```bash +git clone https://github.com/Kaos599/jev-writer +cp -r jev-writer/skills/jev-writer ~/.claude/skills/ +``` + +Then ask your agent to analyse your writing, and it will pick the skill up. + Reference files cover [providers](skills/jev-writer/references/providers.md), [platform exports](skills/jev-writer/references/platform-exports.md), [interpreting results](skills/jev-writer/references/interpreting-results.md), [dashboard design](skills/jev-writer/references/dashboard-design.md), and [rubric authoring](skills/jev-writer/references/rubric-authoring.md). ## Known limits @@ -182,6 +312,7 @@ Stated plainly, because a tool that hides these is worse than no tool. - **Impressions are author-only.** No third-party API can supply them at any price. They are rendered only into the authenticated author's own session. - **Jev is calibrated, not correct.** Typed output guarantees the interface, not the truth. Validate rubrics against your own labelled examples before trusting them. - **Score magnitudes are ordinal.** TypeSafe warns against interpolating between levels, which is why every correlation here is Spearman rank rather than Pearson. +- **Two posts on the same day cannot be told apart.** The analytics export joins on date. When two posts share one, metrics, post URL and media are all left null rather than guessed. ## FAQ @@ -195,17 +326,20 @@ No. TypeSafe operated a waitlist at launch. The AI Gateway and OpenRouter provid Zero Data Retention is requested on every AI Gateway call. Check your provider's terms, since this tool sends your content to a third party and cannot make guarantees on their behalf. **Why does it refuse to give me findings?** -Because you have fewer than 25 posts with outcome data. At that size, correlations are dominated by noise and any confident answer would be fabricated. Descriptive statistics are still shown. +Because you have fewer than 25 posts with outcome data. At that size, correlations are dominated by noise and any confident answer would be fabricated. Descriptive statistics and the dashboard are still produced. **Can I use a normal LLM instead of Jev?** Partly. The AI SDK's `evaluate` works with OpenAI, Anthropic, and Google models through an adapter, but those do not return calibrated probability distributions, so the rubric-health check and the confidence gating stop working. +**Is it on npm?** +Not yet. `npx github:Kaos599/jev-writer` installs straight from this repo and works today. + ## Contributing New rubric packs and platform adapters are the most useful contributions. Packs must declare their pre-registration tiers and pass `validatePack`. See [AGENTS.md](AGENTS.md) for conventions and the traps worth knowing about. ```bash -npm test # 27 tests +npm test # 67 tests node src/cli.mjs doctor # validates every pack, makes one live call ``` diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..eec154a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,25 @@ +# Security + +## Reporting a vulnerability + +Open a [private security advisory](https://github.com/Kaos599/jev-writer/security/advisories/new). Please do not open a public issue for anything exploitable. + +## What this tool does with your data + +Worth stating plainly, because the whole point of the tool is to read things you wrote. + +**Your posts are sent to a third party.** Every rating is an API call carrying the text of one post to whichever provider your key belongs to: Vercel AI Gateway, OpenRouter, or TypeSafe. Zero Data Retention is requested on AI Gateway calls. This project cannot make guarantees on a provider's behalf, so read their terms. + +**Nothing is sent anywhere else.** There is no telemetry, no analytics, no phone-home, and no network call the CLI makes that is not a rating request to your chosen provider. + +**Everything else stays local.** Exports are parsed on your machine. `corpus.jsonl`, `ratings.jsonl`, `report.json` and `dashboard.html` are written to `jev-out/` and never transmitted. + +## Things not to commit + +`report.json` and `dashboard.html` both embed the full text of every post in your corpus. `jev-out/` is gitignored for that reason. Check before you attach either to an issue. + +Keys are read from the environment. The CLI never writes a key to disk and never prints one, including in error output. Use `.env.example` as a template and keep your real `.env` out of git. + +## Supported versions + +Pre-1.0. Fixes land on the latest release only. diff --git a/llms.txt b/llms.txt index 768e2f3..3a73a5c 100644 --- a/llms.txt +++ b/llms.txt @@ -10,10 +10,38 @@ withholds conclusions when the sample is too small to support them. ## What it is -- A command-line tool and agent skill, MIT licensed, Node >= 20. +- A command-line tool and agent skill, MIT licensed, Node >= 20, 67 tests. - Bring your own key: Vercel AI Gateway, OpenRouter, or TypeSafe direct. - LinkedIn is the first adapter; rubric packs make other platforms user-space files. +## Running it + +The package is not published to npm. Both of these work: + +``` +npx github:Kaos599/jev-writer doctor +npx github:Kaos599/jev-writer run ./exports +``` + +Or from a clone: `git clone https://github.com/Kaos599/jev-writer`, `npm +install`, then `node src/cli.mjs doctor` and `node src/cli.mjs run ./exports`. +`npx jev-writer` alone does not resolve. + +Commands: `doctor` (node, keys, packs, one live call), `build ` (parse +exports into a corpus), `rate` (rate the corpus, resumable), `analyze` +(correlate against outcomes, write the report), `dashboard` (render the report +as one HTML page), `run ` (all of them in order). + +Output goes to `jev-out/`, overridable with `JEV_WRITER_OUT`: `corpus.jsonl`, +`ratings.jsonl`, `report.json`, `dashboard.html`. The last two embed the full +text of every post, so the directory is gitignored. + +The agent skill installs with: + +``` +npx skills add Kaos599/jev-writer --skill jev-writer +``` + ## What makes it different from an LLM rating your posts - Jev returns calibrated probability distributions rather than generated scores, @@ -25,7 +53,10 @@ withholds conclusions when the sample is too small to support them. shows descriptive statistics only. - Headline results are re-tested as partial correlations controlling for publication date, because a rated dimension and an outcome that both drift - over time will correlate for no reason. + over time will correlate for no reason. The false-discovery correction runs on + the controlled p-value, not the raw one. +- Permutation tests and bootstrap intervals are seeded and the seed is reported, + so any finding is reproducible by anyone holding the same data. - Countable features such as length, emoji, hashtags and posting cadence are computed in code and compete with the model's judgments on equal footing. @@ -36,10 +67,26 @@ withholds conclusions when the sample is too small to support them. - conversion rate = engagements divided by impressions. Given that people saw it, did the writing work? This is unconfounded by follower count and feed luck. +## Importing it as a library + +The `exports` map in `package.json` is the stable surface; deep paths into +`src/` do not resolve from outside the package. + +- `jev-writer` - everything below, re-exported +- `jev-writer/stats.mjs` - correlations, permutation tests, the power gate +- `jev-writer/report.mjs` - the plain-language report layer +- `jev-writer/dashboard.mjs` - `buildDashboard(report, opts)`, returns HTML +- `jev-writer/rubrics/pack.mjs` - pack format and `validatePack` +- `jev-writer/rubrics/linkedin-post.mjs` - the shipped pack and its directions +- `jev-writer/adapters/linkedin.mjs` - LinkedIn export parsing + ## Key files - README: https://github.com/Kaos599/jev-writer#readme +- Contributor notes and known traps: AGENTS.md +- Agent skill and its five reference files: skills/jev-writer/ - Rubric pack format and validation: src/rubrics/pack.mjs - Statistics and the power gate: src/stats.mjs - Plain-language report layer: src/report.mjs +- Dashboard renderer: src/dashboard.mjs - LinkedIn export adapter: src/adapters/linkedin.mjs diff --git a/package-lock.json b/package-lock.json index e747f7b..bc1aac4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,18 @@ { - "name": "content-signal", + "name": "jev-writer", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "content-signal", + "name": "jev-writer", "version": "0.1.0", "license": "MIT", "dependencies": { "ai": "^7.0.107" }, "bin": { - "content-signal": "src/cli.mjs" + "jev-writer": "src/cli.mjs" }, "engines": { "node": ">=20" diff --git a/package.json b/package.json index 83278b4..43601fb 100644 --- a/package.json +++ b/package.json @@ -1,25 +1,76 @@ { "name": "jev-writer", "version": "0.1.0", - "description": "Find out which qualities of your writing actually predict engagement - with pre-registered rubrics, calibrated model judgments, and statistics honest enough to tell you when your sample is too small to know.", + "description": "Find out which qualities of your writing actually predict engagement. Rates your published posts against a pre-registered rubric using Jev's calibrated judgments, then tests those ratings against your real engagement data - with a power gate, date-confound control and false-discovery correction that make it refuse a conclusion your sample cannot support.", "keywords": [ - "content-analytics", "linkedin-analytics", "writing-analysis", "engagement-analysis", - "creator-analytics", "social-media-analytics", "typesafe", "jev", "system-one", - "structured-outputs", "llm-evaluation", "rubric", "pre-registration", - "spearman", "statistical-power", "openrouter", "ai-gateway", "claude-skill", "agent-skill" + "content-analytics", + "linkedin-analytics", + "writing-analysis", + "engagement-analysis", + "creator-analytics", + "social-media-analytics", + "llm-as-a-judge", + "llm-evaluation", + "evals", + "typesafe", + "jev", + "system-one", + "structured-outputs", + "calibrated-probabilities", + "rubric", + "pre-registration", + "statistical-power", + "spearman", + "false-discovery-rate", + "openrouter", + "ai-gateway", + "claude-skill", + "agent-skills", + "cli" ], "license": "MIT", "type": "module", - "bin": { "jev-writer": "./src/cli.mjs" }, - "files": ["src", "skills", "docs", "README.md", "LICENSE"], - "engines": { "node": ">=20" }, - "repository": { "type": "git", "url": "git+https://github.com/Kaos599/jev-writer.git" }, + "bin": { + "jev-writer": "./src/cli.mjs" + }, + "files": [ + "src", + "skills", + "README.md", + "LICENSE", + "CHANGELOG.md", + "AGENTS.md", + "llms.txt" + ], + "engines": { + "node": ">=20" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/Kaos599/jev-writer.git" + }, "homepage": "https://github.com/Kaos599/jev-writer#readme", - "bugs": { "url": "https://github.com/Kaos599/jev-writer/issues" }, + "bugs": { + "url": "https://github.com/Kaos599/jev-writer/issues" + }, "scripts": { "test": "node --test test/*.test.mjs" }, "dependencies": { "ai": "^7.0.107" + }, + "exports": { + ".": "./src/index.mjs", + "./rubrics/pack.mjs": "./src/rubrics/pack.mjs", + "./rubrics/linkedin-post.mjs": "./src/rubrics/linkedin-post.mjs", + "./stats.mjs": "./src/stats.mjs", + "./report.mjs": "./src/report.mjs", + "./dashboard.mjs": "./src/dashboard.mjs", + "./adapters/linkedin.mjs": "./src/adapters/linkedin.mjs", + "./package.json": "./package.json" + }, + "author": "Kaos599", + "publishConfig": { + "access": "public" } } diff --git a/skills/jev-writer/SKILL.md b/skills/jev-writer/SKILL.md index 91c072f..5dc5d0f 100644 --- a/skills/jev-writer/SKILL.md +++ b/skills/jev-writer/SKILL.md @@ -2,11 +2,11 @@ name: jev-writer description: > Analyse a person's published writing to find which qualities actually predict - engagement, then build them a dashboard and a personalised review prompt. Use + engagement, then give them a dashboard and a personalised review prompt. Use when someone wants to know why some of their posts do better than others, wants their content audited or graded, asks what to change about their writing, or wants a content-analytics dashboard built from their own data. Walks them - through exporting their data and getting an API key, runs the analysis, and + through exporting their data and getting an API key, runs the pipeline, and refuses to report findings the sample cannot support. Analyses existing posts; it does not write them. license: MIT @@ -16,7 +16,7 @@ license: MIT This skill turns "why do some of my posts work?" into an answer grounded in that person's own data. You will walk them through two downloads and one key, run a -pipeline, and then build them something legible. +pipeline, and then help them read what comes out. **The whole value is honesty.** A dozen tools already tell people their content is great. This one tells them when their sample is too small to know, when a @@ -30,12 +30,12 @@ Say this early and plainly if the user seems to expect otherwise: - **It does not write or rewrite posts.** Jev is a System One model and is not trained to generate text. It returns typed judgments and probabilities. - **It cannot recover engagement data the platform no longer exposes.** On - LinkedIn, impressions exist for roughly the last 12 months and the top ~50 + LinkedIn, impressions exist for roughly the last 12 months and the top 50 posts. Older engagement is gone for everyone, including the platform's own UI. - **It cannot establish causation.** Everything it finds is an association in one person's history, with unmeasured confounders. -## Step 1 — get their data +## Step 1. Get their data Do not start rating anything until real data is in hand. For LinkedIn the user needs **both** of these, and they must be told the first one takes hours: @@ -43,7 +43,7 @@ needs **both** of these, and they must be told the first one takes hours: 1. **The larger data archive.** Settings → Data Privacy → *Get a copy of your data* → **"Download larger data archive"**. The basic archive contains no `Shares_*.csv` at all, which is the single most common failure. This gives - post text, dates and URLs, and **no engagement data whatsoever** — the + post text, dates and URLs, and **no engagement data whatsoever**. The `Reactions.csv` and `Comments.csv` inside it record what they did on *other people's* posts. 2. **Post analytics export.** Creator Mode on → Analytics & Tools → Post @@ -51,13 +51,15 @@ needs **both** of these, and they must be told the first one takes hours: impressions. Tell them to trigger the archive first, because everything else is blocked on it. -Put both files in one directory. +Put both files, still zipped, in one directory. Never suggest scraping, browser automation against the platform, or third-party data vendors. The first-party route is free, complete enough, and does not risk their account. -## Step 2 — get a key +Other platforms are covered in `references/platform-exports.md`. + +## Step 2. Get a key Any one of three works, and the tool auto-detects which is present: @@ -73,111 +75,161 @@ hundred posts costs cents. Check current pricing before a large run. Never ask the user to paste a key into the chat. Have them set it in their environment or a local `.env`, then run `doctor`. -## Step 3 — run it +Details per provider in `references/providers.md`. + +## Step 3. Run it + +The package is not published to npm. These are the invocations that work: + +```bash +npx github:Kaos599/jev-writer doctor # node, key, pack validation, one live call +npx github:Kaos599/jev-writer run ./exports # build, rate, analyse, render +``` + +Or from a clone, which is what you want if you plan to edit a rubric: ```bash -npx jev-writer doctor # node, key, pack validation, one live call -npx jev-writer run ./exports # build corpus, rate it, analyse it +git clone https://github.com/Kaos599/jev-writer && cd jev-writer +npm install +node src/cli.mjs doctor +node src/cli.mjs run ./exports ``` +`npx jev-writer` on its own does not resolve. Do not write it. + `doctor` first, always. It catches a missing key, an `ai` package too old for the Gateway path, and a broken rubric before a full run spends anything. -Outputs land in `jev-writer-out/`: `corpus.jsonl`, `ratings.jsonl`, and -`report.json`. **`report.json` is the artifact you build everything else from.** +### The commands -## Step 4 — read the report honestly +| Command | What it does | +| --- | --- | +| `doctor` | check Node, keys, every pack; make one live call | +| `build ` | parse the exports in `` into `corpus.jsonl` | +| `rate` | rate the corpus, resumable after an interruption | +| `analyze` | correlate against outcomes, write `report.json` | +| `dashboard` | render `report.json` to `dashboard.html` | +| `run ` | build, rate, analyze, dashboard, in order | + +Everything lands in `jev-out/`, overridable with the `JEV_WRITER_OUT` +environment variable: `corpus.jsonl`, `ratings.jsonl`, `report.json`, +`dashboard.html`. + +**`report.json` is the artifact everything else is built from.** Both it and +`dashboard.html` embed the full text of every post. `jev-out/` is gitignored; +never commit either file, and never paste a whole corpus into a chat. + +If `rate` is interrupted, run it again. It reads `ratings.jsonl` and skips posts +already done rather than paying for them twice. + +## Step 4. Read the report honestly `report.json` has already done the interpretation. It contains no p-values on its main surface, and you should not reintroduce any. -- `findings[]` — each has a `sentence` written in plain English, and a - `confirmed` flag. **Only `confirmed: true` findings are findings.** Everything - else is `exploratory`: present it as "worth watching", never as a conclusion. -- `power` — if `verdict` is `descriptive`, the sample is too small and **no - correlational claim may be made at all**. Say so directly: "you have N posts - with engagement data, which is not enough to tell signal from noise yet." - Offer descriptives and suggest re-running after more posts. -- `weakSpots[]` — the gap between their typical post and their own +- `findings[]` each carry a `sentence` written in plain English and a `confirmed` + flag. **Only `confirmed: true` findings are findings.** Everything else is + exploratory: present it as "worth watching", never as a conclusion. +- `power` is keyed by outcome. If a `verdict` is `descriptive`, the sample is too + small and **no correlational claim may be made for that outcome at all**. Say + so directly: "you have N posts with engagement data, which is not enough to + tell signal from noise yet." Offer descriptives and suggest re-running after + more posts. +- `weakSpots[]` is the gap between their typical post and their own best-performing posts. This is the most actionable section; lead with it. -- `posts[].grade` and `posts[].bands` — per-post grades and direction-aware +- `posts[].grade` and `posts[].bands` are per-post grades and direction-aware bands. **Never recompute a band or assume a high value is good.** Direction is - metadata on the pack; `ai_generated_feel` high is bad, `specificity` high is + metadata on the pack: high `ai_generated_feel` is bad, high `specificity` is good. -- `writingPrompt` — hand this to them verbatim. It is generated from their own - results and is the single most useful output for their next draft. +- `writingPrompt` should be handed over verbatim. It is generated from their own + results and is the single most useful output for their next draft. It is + `null` when nothing validated, and that null is honest information rather than + a bug. If a rubric's mean confidence across the corpus is near 0.5, its levels do not discriminate. Flag it rather than quoting its number. -## Step 5 — build them a dashboard - -The user almost always wants to *see* this. Build a dashboard from -`report.json`. You are the renderer; the data is already interpreted. - -### Non-negotiable presentation rules - -These come from a real user rejecting an earlier version as unreadable: - -1. **No statistics vocabulary on the main surface.** No "rho", "p =", "CI", - "correlation", "significant", "n=". Those belong in one collapsed "how this - was worked out" section at the bottom, or nowhere. -2. **Bands and colour, not raw numbers.** A rubric value of 1.10 means nothing - to anyone. Show `bands[dim].label` (Poor / Weak / Typical / Strong / - Excellent) with colour: green for excellent and strong, grey for typical, - amber for weak, red for poor. The band is already direction-aware — render - it, do not derive it. -3. **Lead with sentences, not charts.** `findings[].sentence` is the headline. - Charts only where they carry something a sentence cannot, such as a volume - collapse over years. -4. **Grade every post 0–100** with a plain word attached: Strong, Good, Mixed, - Weak. -5. **Every percentage gets a one-line explanation** of what it means. "Reach" and - "engagement rate" are not self-evident. -6. **No scatter plots of rubric values against outcomes.** That is the part a - real user called meaningless. - -### Suggested structure - -1. How your content is doing — volume by year, median reach and engagement with - plain explanations, follower growth. -2. What's working for you — confirmed findings as large statements, then - `weakSpots` as "the gap between your best posts and your typical post". -3. Your posts, graded — sortable, grade-first, click to expand into full text and - all judgments as coloured bands with their level descriptions in words. -4. Your writing prompt — `report.writingPrompt` in a copy-to-clipboard box. -5. How this was worked out — collapsed; methodology and caveats live here. - -### Build guidance - -Prefer a **single self-contained HTML file** with inline CSS and JS and the data -embedded, so it opens by double-click with no server and no build step. If the -user already has a React or Next.js project and asks for it there, shadcn/ui -components map cleanly onto this structure: `Card` for each post, `Badge` for -band chips, `Table` for the graded list, `Tabs` for the sections, `Collapsible` -for the methodology. Otherwise implement the same visual conventions — subtle -borders, rounded corners, muted foregrounds, badge chips — in plain CSS. - -Dark-mode-first, working in both themes via CSS custom properties. Tabular -numerals for figures. Phone width with no horizontal scroll. - -Write a small generator script alongside the HTML so the dashboard can be -regenerated when data changes, rather than being a dead artifact. +Full guidance in `references/interpreting-results.md`. + +## Step 5. Give them the dashboard + +The user almost always wants to *see* this. Do not hand-write a page. The tool +renders one: + +```bash +node src/cli.mjs dashboard # or: npx github:Kaos599/jev-writer dashboard +open jev-out/dashboard.html +``` + +`run` already does this as its last step, so after a full run the file is +waiting. It is one self-contained HTML file: inline CSS, inline JS, data +embedded, no CDN and no fetch, so it opens by double-click with no network. A +178-post corpus renders to about 600 KB. + +It reads `report.json` rather than recomputing anything, so what the user sees is +exactly what the report layer concluded. If `report.json` is missing, the command +tells you to run `analyze` first. + +Your job after rendering it is to walk them through it and keep the reading +honest: point at `weakSpots`, name which findings are confirmed and which are +only exploratory, and repeat the sample-size caveat if `power` says so. + +### If someone wants a custom dashboard + +Only when they ask for it inside their own React or Next.js project, or want a +different shape. `references/dashboard-design.md` is the specification. It +records the presentation rules the shipped dashboard follows and why each one +exists, including the user feedback that produced them. Build from `report.json` +only; the data is already interpreted. + +The rules that matter most, in short: no statistics vocabulary on the main +surface, bands and colour rather than raw rubric numbers, lead with +`findings[].sentence`, grade every post 0 to 100 with a plain word attached, +explain every percentage, no scatter plots of rubric values against outcomes, and +de-emphasise exploratory results so a skimming reader cannot mistake one for a +finding. + +`buildDashboard` is also exported, so a custom renderer can start from the +shipped page rather than from nothing: + +```js +import { buildDashboard } from 'jev-writer/dashboard.mjs'; + +const html = buildDashboard(report, { ratings, pack, platform: 'LinkedIn' }); +``` + +`ratings` is the parsed rows of `ratings.jsonl`, `pack` is the rubric pack, and +`platform` is a display name. All three are optional. ## Extending to other platforms A rubric pack is a versioned set of typed questions plus the state fields each question may see. To support a new platform, write an adapter producing items -with text plus an outcome, and a pack. See `src/rubrics/pack.mjs`. +with text plus an outcome, and a pack. The format lives in `src/rubrics/pack.mjs` +in the repo, and consumers import it as `jev-writer/rubrics/pack.mjs`. Two rules when authoring a pack: - **Never ask the model anything code can count.** Length, emoji, hashtags, cadence and media type are computed exactly and compete with the judgments. - **Pre-register every question** as `primary` or `exploratory` before seeing - results, and declare a `direction` for each. Both are enforced by - `validatePack` and the report layer. + results, and give every rated dimension a direction of merit. `validatePack` + enforces the tier. Directions are a separate `directions` export alongside the + pack; the report layer throws when one is missing, so a pack without them + produces no `report.json` and no dashboard. Rubric wording is load-bearing. In development, tightening one clause moved a judgment from 0.99 to 0.71 on identical input. Treat pack edits as breaking changes and re-rate rather than pooling ratings across wordings. + +Full guidance in `references/rubric-authoring.md`. + +## Reference files + +| File | Covers | +| --- | --- | +| `references/providers.md` | getting a key, the three transports, cost, adding a fourth | +| `references/platform-exports.md` | what is exportable from X, Medium, Substack, YouTube, Reddit, Instagram | +| `references/interpreting-results.md` | every field in `report.json` and what may be said about it | +| `references/dashboard-design.md` | why the dashboard looks the way it does, and the spec for a custom one | +| `references/rubric-authoring.md` | writing and validating a pack | diff --git a/skills/jev-writer/references/dashboard-design.md b/skills/jev-writer/references/dashboard-design.md index 0390b1e..858c06b 100644 --- a/skills/jev-writer/references/dashboard-design.md +++ b/skills/jev-writer/references/dashboard-design.md @@ -1,7 +1,20 @@ -# Building the dashboard +# The dashboard: why it looks like this -The dashboard is the deliverable most users actually look at. Build it from -`report.json` only — everything in that file has already been interpreted. +**You do not need to build a dashboard.** `jev-writer dashboard` renders one, and +`run` calls it as its last step: + +```bash +node src/cli.mjs dashboard # or: npx github:Kaos599/jev-writer dashboard +open jev-out/dashboard.html +``` + +It reads `jev-out/report.json` and writes `jev-out/dashboard.html`. If the report +is missing it tells you to run `analyze` first. + +This file is here for two reasons. It records **why** the shipped page is shaped +the way it is, so nobody redesigns it back into the version that failed. And it +is the **specification** if someone genuinely needs a custom one, inside their +own React or Next.js project or in a different shape. ## What got rejected, and why @@ -12,7 +25,7 @@ product. The specific complaints, which are now rules: - It printed `rho` and `p` on the main surface. The user asked, reasonably, "what is rho?" If a reader has to look up a symbol, the number should not have been shown. -- It showed raw rubric values like `1.10` on a 0–3 scale with no anchor, so +- It showed raw rubric values like `1.10` on a 0 to 3 scale with no anchor, so nothing could be judged good or bad. - It led with scatter plots of rubric values against outcomes, which communicate nothing to someone who does not already know what a correlation looks like. @@ -21,50 +34,83 @@ product. The specific complaints, which are now rules: ## Non-negotiable rules +The shipped page follows all of these. A custom one must too. + 1. **No statistics vocabulary on the main surface.** Not `rho`, `p =`, `CI`, `correlation`, `Spearman`, `significant`, `n=`. These may appear only inside one collapsed "How this was worked out" section at the bottom. 2. **Bands and colour, never raw rubric numbers.** Render `bands[dim].label` (Poor / Weak / Typical / Strong / Excellent). Green for excellent and strong, - neutral grey for typical, amber for weak, red for poor. The band is already - direction-aware — render it, never recompute it, never assume high is good. + neutral grey for typical, amber for weak, red for poor. The band arrives + direction-aware from the report. Render it, never recompute it, never assume + high is good. 3. **Lead with sentences.** `findings[].sentence` is the headline. It is already written for a human. -4. **Grade every post 0–100** with a plain word attached: Strong, Good, Mixed, - Weak. +4. **Grade every post 0 to 100** with a plain word attached: Strong, Good, Mixed, + Weak. `posts[].grade` is `null` when nothing validated; show the post without + a grade rather than inventing one. 5. **Explain every percentage.** "Reach" and "engagement rate" are not self-evident; each needs one line saying what it means. 6. **No scatter plots of rubric values against outcomes.** Charts only where they - carry something a sentence cannot — a volume collapse across years, follower - growth over time. + carry something a sentence cannot, such as a volume collapse across years or + follower growth over time. 7. **Exploratory results are visually de-emphasised and labelled.** A user skimming must not be able to mistake one for a finding. +8. **Narrative sentences are derived, not templated.** An earlier version + hard-coded "you stopped posting", which was true of exactly one corpus and + reads as a false accusation against anyone whose volume grew. `corpus.cadence` + in the report carries the derived answer; use it. ## Structure that works -1. **How your content is doing** — volume by year, median reach and engagement +This is the order the shipped page uses. + +1. **How your content is doing.** Volume by year, median reach and engagement with plain explanations, follower growth. -2. **What's working for you** — confirmed findings as large readable statements, +2. **What's working for you.** Confirmed findings as large readable statements, then `weakSpots` as "the gap between your best posts and your typical post". This is the section users act on; put it high. -3. **Your posts, graded** — sortable, grade first. Click to expand into the full +3. **Your posts, graded.** Sortable, grade first. Click to expand into the full text, every judgment as a coloured band with its level description in words, and the `fixes` list as "what would most improve this post". -4. **Your writing prompt** — `report.writingPrompt` in a copy-to-clipboard box, +4. **Your writing prompt.** `report.writingPrompt` in a copy-to-clipboard box, with one line explaining it was generated from their own results. -5. **How this was worked out** — collapsed. Methodology, sample size, caveats, - and the only place statistical detail is allowed. +5. **How this was worked out.** Collapsed. Methodology, sample size, caveats, and + the only place statistical detail is allowed. + +Judgments inside the expanded post view are grouped into families. A pack may +declare its own `groups`; the LinkedIn pack declares five, covering all thirty +questions. Grouping is presentation only and never touches how a question is +asked, rated or correlated, and anything a pack does not place falls into a +catch-all so a new pack renders sensibly on day one. + +## Building a custom one -## Implementation +Build from `report.json` only. Everything in that file has already been +interpreted; re-deriving statistics in a renderer is how the rules above get +broken. -Default to a **single self-contained HTML file**: inline CSS and JS, data -embedded as a JS object, no CDN, no fetch, no build step. It must open by -double-clicking with `file://`. Also write a small generator script so the -dashboard can be regenerated when data changes rather than becoming a dead file. +The easiest starting point is the shipped renderer itself, which does no I/O: -If the user already has a React or Next.js project and wants it there, -shadcn/ui maps cleanly onto this structure: `Card` per post, `Badge` for band -chips, `Table` for the graded list, `Tabs` for sections, `Collapsible` for the +```js +import { buildDashboard } from 'jev-writer/dashboard.mjs'; + +const html = buildDashboard(report, { ratings, pack, platform: 'LinkedIn' }); +``` + +`report` is exactly what `buildReport` returned. `ratings` is the parsed rows of +`ratings.jsonl`, used for Choice answers. `pack` supplies question groups and +counts. `platform` is a display name, and leaving it out keeps the copy generic. +All three options are optional. + +If you are writing a page from scratch, default to a **single self-contained HTML +file**: inline CSS and JS, data embedded as a JS object, no CDN, no fetch, no +build step, opening by double-click over `file://`. That is what the shipped page +does, and it is why it works on a laptop with no network. + +If the user already has a React or Next.js project and wants it there, shadcn/ui +maps cleanly onto this structure: `Card` per post, `Badge` for band chips, +`Table` for the graded list, `Tabs` for sections, `Collapsible` for the methodology, `Progress` for a grade bar. Do not introduce a build step into a project that does not already have one. @@ -76,14 +122,20 @@ Dark-mode-first, working in both themes. Define colours as custom properties on an explicit toggle. Give `body` an explicit background. Subtle borders, rounded corners, muted foreground for secondary text, badge chips -for bands. Generous whitespace. Strong typographic hierarchy — the grade and the -findings should be the largest things on the page. Tabular numerals +for bands. Generous whitespace. Strong typographic hierarchy, so the grade and +the findings are the largest things on the page. Tabular numerals (`font-variant-numeric: tabular-nums`) for every figure so columns align. Must work at phone width with a 16px gutter and no horizontal scroll. ## Colour and accessibility -Do not encode meaning in colour alone — every band carries its word as well as -its colour, so the page survives greyscale and colour blindness. Keep contrast at +Do not encode meaning in colour alone. Every band carries its word as well as its +colour, so the page survives greyscale and colour blindness. Keep contrast at 4.5:1 for body text in both themes. + +## Privacy + +The page embeds the full text of every post. It is written to `jev-out/`, which +is gitignored, and it is generated rather than committed. A custom renderer +inherits that obligation. diff --git a/skills/jev-writer/references/interpreting-results.md b/skills/jev-writer/references/interpreting-results.md index e115815..74c8477 100644 --- a/skills/jev-writer/references/interpreting-results.md +++ b/skills/jev-writer/references/interpreting-results.md @@ -1,8 +1,9 @@ # Reading `report.json` without overstating it -`report.json` is the only file you should build a dashboard, a summary or advice -from. The statistics have already been run and translated; your job is to render -the conclusions, not to re-derive them. +`jev-out/report.json` is the only file you should build a summary or advice from. +The statistics have already been run and translated; your job is to render the +conclusions, not to re-derive them. The shipped dashboard reads this same file +and nothing else. The single most important rule: **this tool's value is that it refuses to flatter.** Dozens of products already tell people their content is great. If you @@ -12,36 +13,51 @@ turned it into one of them. ## The shape ``` +generatedAt ISO timestamp +pack { id, version } summary posts, rated, withOutcomes, dateRange -power per outcome: verdict, criticalR, minimumDetectableEffect, headline -findings[] sentence, confirmed, dimension, outcome, comparison, direction -weakSpots[] dimension, advice, typical, winners, gap -validated[] the dimensions that actually predicted something -posts[] grade {score, parts}, bands {per dimension}, fixes[] +power per outcome: n, nTests, criticalR, minimumDetectableEffect, + expectedFalsePositives, verdict, headline +outcomes the outcome definitions, with labels and explainers +findings[] sentence, confirmed, dimension, dimensionLabel, outcome, + outcomeLabel, comparison, direction, tier +validated[] { dimension, weight } for dimensions that predicted something +weakSpots[] dimension, label, direction, typical, winners, gap, gapShare, advice bandEdges the quintile cut points used, per dimension +corpus volumeByYear, cadence, followers, medians, totals +posts[] every corpus field, plus grade, bands {per dimension}, fixes[] writingPrompt a ready-to-paste string, or null ``` +Findings also carry a `_stats` object. It exists for the collapsed methods +appendix only. Anything in it is statistics vocabulary and must not reach the +main surface. + ## `power` decides what you are allowed to say -Check this before quoting anything. +`power` is keyed by outcome, so check the one you are about to quote. -- **`verdict: 'descriptive'`** — fewer than 25 posts carry outcome data. No - correlational claim may be made at all. Say so plainly: *"you have N posts with - engagement data, which isn't enough to separate signal from noise yet."* Show - descriptives, suggest re-running later. Do not hedge your way into a finding. -- **`verdict: 'weak'`** — between 25 and 60. Findings are permitted but only the +- **`verdict: 'descriptive'`** means fewer than 25 posts carry data for that + outcome. No correlational claim may be made for it at all. Say so plainly: + *"you have N posts with engagement data, which isn't enough to separate signal + from noise yet."* Show descriptives, suggest re-running later. Do not hedge + your way into a finding. +- **`verdict: 'weak'`** is between 25 and 60. Findings are permitted but only the pre-registered ones, and the sample-size caveat belongs somewhere visible. -- **`verdict: 'report'`** — 60 or more. False-discovery correction is applied. +- **`verdict: 'report'`** is 60 or more. + +False-discovery correction is applied in every case where correlations run at +all, not only above 60. -## `findings[]` — only `confirmed` ones are findings +## `findings[]`: only `confirmed` ones are findings Each finding carries a `sentence` already written in plain English. Use it verbatim where you can; it was written to avoid statistics vocabulary. - `confirmed: true` means the question was **pre-registered** as a primary - hypothesis before any result was seen, and it survived the significance test - after controlling for publication date. These are evidence. + hypothesis before any result was seen, and it survived false-discovery + correction on the p-value computed *after* controlling for publication date. + These are evidence. - `confirmed: false` means exploratory. These are **hypotheses for the next batch of data**, not conclusions. Render them de-emphasised, labelled, and never in a headline. If the user asks "so what should I change", answer only from @@ -49,22 +65,29 @@ verbatim where you can; it was written to avoid statistics vocabulary. Findings are sorted confirmed-first, then by effect size. -## `weakSpots[]` — lead with this +## `weakSpots[]`: lead with this This is the most actionable section and the one users respond to. Each entry compares a **typical** post against that person's **own best-performing** posts -on one dimension, and `advice` is a finished sentence. +on one dimension, and `advice` is a finished sentence. `gap` is already +direction-aware, so a positive gap always means the winners do more of the good +thing, whichever way the dimension runs. + +Weak spots are only computed over dimensions that validated against an outcome, +and only when there are at least nine posts with a conversion rate to draw a top +third from. An empty array is a legitimate result. An earlier version of this file counted how many posts fell in the bottom two -bands. That was meaningless — bands are quintiles of the person's own +bands. That was meaningless: bands are quintiles of the person's own distribution, so the bottom two are 40% of posts by construction and every dimension scored exactly 40%. If you ever find yourself computing "share of posts in a low band", you have reintroduced that bug. -## `posts[].bands` — render, never derive +## `posts[].bands`: render, never derive -Every dimension is banded Poor / Weak / Typical / Strong / Excellent against the -author's own distribution, and **the band is already direction-aware**. +Every rated dimension is banded Poor / Weak / Typical / Strong / Excellent +against the author's own distribution, and **the band is already +direction-aware**. This matters: a high `ai_generated_feel` is bad, a high `specificity` is good. If you colour by raw value you will paint the worst posts green. Use @@ -74,16 +97,22 @@ user sees. `neutral` dimensions such as `technicality` are descriptive rather than better or worse. Do not colour them as though more is better. +Choice questions are not banded. They have no ordering to band, so they do not +appear in `bands` and are read from `ratings.jsonl` instead. + ## `posts[].grade` -A 0–100 score built **only from dimensions validated against this person's own -outcomes** — not from all 30. A composite over everything would be a vibe; a -composite over what actually moved their numbers is a prediction. +A 0 to 100 score built **only from dimensions validated against this person's own +outcomes**, not from every rated dimension. A composite over everything would be +a vibe; a composite over what actually moved their numbers is a prediction. `grade` is `null` when nothing validated. That is a legitimate outcome, usually because the sample was too small. Show the posts without grades rather than inventing a score. +`fixes[]` on each post names up to three validated dimensions where that post +sits below Strong. It is the honest answer to "what would most improve this one". + ## `writingPrompt` Hand this to the user verbatim, in a copy-to-clipboard box. It is generated from @@ -91,15 +120,33 @@ their own results: the dimensions that predicted engagement for them, what to ai for in each, and where their best posts differ from their typical one. It is `null` when there were no findings. Do not substitute generic writing -advice — the absence of a prompt is itself honest information. +advice. The absence of a prompt is itself honest information. + +## `corpus` + +The part that needs no model and no statistics: `volumeByYear`, `followers`, +median reach and conversion rates, totals. `corpus.cadence` describes how posting +volume actually moved (`rising`, `steady`, `declined`, `collapsed`) and is `null` +when there is less than two years of history. Derive the sentence from it rather +than asserting a trend. An earlier dashboard hard-coded "you stopped posting", +which was true of exactly one corpus. ## Rubric health If a dimension's mean confidence across the corpus sits near 0.5, its levels do -not discriminate and its number should not be quoted. This is a property no +not discriminate and its number should not be quoted. Confidence lives on each +row in `ratings.jsonl`, not in `report.json`. This is a property no prompt-and-parse pipeline can measure, and it is worth surfacing: it tells the user which parts of the rubric to rewrite. +## Corpus warnings + +`build` prints warnings that never reach `report.json`: unparseable dates in an +export, a follower total that fails its consistency check, days carrying more +than one post. Any of these means some fields are null on purpose. If a user asks +why a post has no metrics, the answer is usually in that output rather than in +the report. + ## Things to say out loud - Associations in one person's history, not causation. diff --git a/skills/jev-writer/references/platform-exports.md b/skills/jev-writer/references/platform-exports.md index f8e1616..19a0eea 100644 --- a/skills/jev-writer/references/platform-exports.md +++ b/skills/jev-writer/references/platform-exports.md @@ -1,6 +1,13 @@ -# Exporting Your Own Posts and Engagement Data — Platform Reference +# Exporting your own posts and engagement data -Status as of **September 2026**. Export UI paths, field names, and API terms on all these platforms change frequently and without notice — treat every "exact filename" claim below as verified against the cited source on the date of that source, not as a permanent guarantee. Where a source could not be confirmed, this is called out explicitly rather than guessed. +**Only LinkedIn ships an adapter.** Everything below is a survey of what a person +can actually get out of each platform, so you can judge whether an adapter and a +rubric pack are worth writing. The pipeline needs items with text plus at least +one outcome; where the table says engagement is absent from the export, the +analysis stops at descriptives. The LinkedIn route is in `SKILL.md`, and writing +a pack is covered in `rubric-authoring.md`. + +Status as of **September 2026**. Export UI paths, field names, and API terms on all these platforms change frequently and without notice - treat every "exact filename" claim below as verified against the cited source on the date of that source, not as a permanent guarantee. Where a source could not be confirmed, this is called out explicitly rather than guessed. --- @@ -18,38 +25,38 @@ window.YTD.tweets.part0 = [ { "tweet": { ... } }, ... ] ``` [Tweet Archivist, 2026](https://www.tweetarchivist.com/twitter-archive-format-explained); confirmed independently by [seramo/twitter-archive-docs](https://github.com/seramo/twitter-archive-docs) and multiple parser repos ([alkihis/twitter-archive-reader](https://github.com/alkihis/twitter-archive-reader/blob/master/Files_to_structures.md), [kltownsend/tweet2csv](https://github.com/kltownsend/tweet2csv)). -**Filename note (gotcha):** The tweet file has been named both `tweets.js` and `tweet.js` across different archive-format eras (older exports moved data around; see [django-ditto issue #229](https://github.com/philgyford/django-ditto/issues/229), which documents X changing the file/directory structure between 2022 exports). **Do not hardcode a single filename** — check for both `data/tweets.js` and `data/tweet.js`, and possibly a `data/tweets-part0.js`/multi-part split for large accounts. +**Filename note (gotcha):** The tweet file has been named both `tweets.js` and `tweet.js` across different archive-format eras (older exports moved data around; see [django-ditto issue #229](https://github.com/philgyford/django-ditto/issues/229), which documents X changing the file/directory structure between 2022 exports). **Do not hardcode a single filename** - check for both `data/tweets.js` and `data/tweet.js`, and possibly a `data/tweets-part0.js`/multi-part split for large accounts. **Fields inside each tweet object** (per multiple sources, cross-checked against a real sample in [gist: bitsgalore](https://gist.github.com/bitsgalore/cfdff3ce67f1ffa85f67e87c778a9e75)): - `id`, `id_str`, `created_at`, `full_text`, `lang`, `source`, `truncated`, `favorited`, `retweeted` -- `favorite_count`, `retweet_count` — **present as fields, but reported to be zeroed out / not real engagement counts in the archive** ("Twitter doesn't include actual engagement metrics in archives" — [Tweet Archivist, 2026](https://www.tweetarchivist.com/twitter-archive-format-explained)). Treat any non-zero value in these fields with caution and do not rely on the archive for engagement history. +- `favorite_count`, `retweet_count` - **present as fields, but reported to be zeroed out / not real engagement counts in the archive** ("Twitter doesn't include actual engagement metrics in archives" - [Tweet Archivist, 2026](https://www.tweetarchivist.com/twitter-archive-format-explained)). Treat any non-zero value in these fields with caution and do not rely on the archive for engagement history. - `entities.hashtags[]`, `entities.user_mentions[]`, `entities.urls[]`, `extended_entities.media[]` - `in_reply_to_status_id`, `in_reply_to_status_id_str`, `in_reply_to_screen_name` - `edit_info.initial.{editTweetIds, editableUntil, editsRemaining, isEditEligible}` (edit history, added ~2022) **What is NOT in the archive:** impressions, reach, or any performance/analytics data. The archive is a content/metadata backup only. [Tweet Archivist, 2026](https://www.tweetarchivist.com/twitter-archive-format-explained) -**Other files in `data/`:** `account.js`, `profile.js`, `follower.js`, `following.js`, `direct-messages.js`, `like.js`, `ad-impressions.js`, `personalization.js`. Note the archive **includes likes but not bookmarks** — a long-standing gap as of 2026. [Tweet Archivist](https://www.tweetarchivist.com/twitter-archive-format-explained) +**Other files in `data/`:** `account.js`, `profile.js`, `follower.js`, `following.js`, `direct-messages.js`, `like.js`, `ad-impressions.js`, `personalization.js`. Note the archive **includes likes but not bookmarks** - a long-standing gap as of 2026. [Tweet Archivist](https://www.tweetarchivist.com/twitter-archive-format-explained) -**Parser gotcha:** Nested JSON-in-JS wrapper must be stripped (`.replace(/^window\.YTD\.\w+\.part\d+\s*=\s*/, '')`) before `JSON.parse`. Numeric-looking fields (`id`, `favorite_count`, `retweet_count`, `indices[]`) are sometimes serialized **as strings**, not numbers — confirmed in the raw sample above (`"favorite_count" : "16"`, `"indices" : ["202","211"]`). Cast explicitly. +**Parser gotcha:** Nested JSON-in-JS wrapper must be stripped (`.replace(/^window\.YTD\.\w+\.part\d+\s*=\s*/, '')`) before `JSON.parse`. Numeric-looking fields (`id`, `favorite_count`, `retweet_count`, `indices[]`) are sometimes serialized **as strings**, not numbers - confirmed in the raw sample above (`"favorite_count" : "16"`, `"indices" : ["202","211"]`). Cast explicitly. ### 1b. X Analytics (analytics.x.com / analytics.twitter.com) -**Gating:** As of the June 2024 policy change (still in effect through 2026), the **full account-wide analytics dashboard requires X Premium** (from $8/mo) or X Premium+; Basic ($3/mo) does not include it. Free accounts are NOT locked out entirely — tapping the bar-chart icon under an individual post (mobile or desktop) shows impressions/likes/replies/reposts for that one post, for free, with no CSV export. [Swydo, 2026](https://www.swydo.com/blog/x-analytics-metrics); [ficilcom.jp, 2026](https://www.ficilcom.jp/en/blog/how-to-use-x-analytics); [tendX, 2026](https://www.tendx.app/blog/how-to-use-x-twitter-analytics) +**Gating:** As of the June 2024 policy change (still in effect through 2026), the **full account-wide analytics dashboard requires X Premium** (from $8/mo) or X Premium+; Basic ($3/mo) does not include it. Free accounts are NOT locked out entirely - tapping the bar-chart icon under an individual post (mobile or desktop) shows impressions/likes/replies/reposts for that one post, for free, with no CSV export. [Swydo, 2026](https://www.swydo.com/blog/x-analytics-metrics); [ficilcom.jp, 2026](https://www.ficilcom.jp/en/blog/how-to-use-x-analytics); [tendX, 2026](https://www.tendx.app/blog/how-to-use-x-twitter-analytics) **CSV export path (Premium required):** `x.com/i/account_analytics` (or the legacy `analytics.twitter.com`/`analytics.x.com` URLs, which still work) → Content/Tweets tab → set date range → "Export data" → CSV. [opentweet.io](https://opentweet.io/tools/x-analytics-csv-analyzer); [tweetarchivist.com, 2026](https://www.tweetarchivist.com/how-to-use-twitter-analytics-2025) **Fields in the export (per multiple guides, not independently verified against a real downloaded file in this research pass):** Tweet ID/permalink, tweet text, date/time, impressions, engagements, engagement rate, retweets/replies/likes as individual counts, link clicks, profile clicks, media views/engagements, hashtag clicks, detail expands, app opens/installs. [Mailchimp](https://mailchimp.com/resources/how-to-use-twitter-analytics/); [tweetarchivist.com](https://www.tweetarchivist.com/how-to-use-twitter-analytics-2025) -**Date range limit:** The on-screen dashboard is commonly reported as a rolling **28-day window**; CSV export is reported to reach back **up to ~90 days** in some accounts. [tendX, 2026](https://www.tendx.app/blog/how-to-use-x-twitter-analytics); [Tweet Archivist, 2026](https://www.tweetarchivist.com/how-to-use-twitter-analytics-2025). These figures vary by source and are **not from an official X document** in this research pass — flag as **unverified against a primary X source**. +**Date range limit:** The on-screen dashboard is commonly reported as a rolling **28-day window**; CSV export is reported to reach back **up to ~90 days** in some accounts. [tendX, 2026](https://www.tendx.app/blog/how-to-use-x-twitter-analytics); [Tweet Archivist, 2026](https://www.tweetarchivist.com/how-to-use-twitter-analytics-2025). These figures vary by source and are **not from an official X document** in this research pass - flag as **unverified against a primary X source**. -**Not verified:** exact CSV column headers (multiple secondary sources roughly agree, but none is X's own documentation — X does not appear to publish a formal export-schema page as of this research). +**Not verified:** exact CSV column headers (multiple secondary sources roughly agree, but none is X's own documentation - X does not appear to publish a formal export-schema page as of this research). ### 1c. X API v2 for reading your own tweets (2026) X removed its free developer tier in **February 2026**; the platform is now **pay-per-use by default**: ~$0.005 per post read, ~$0.015 per post write ($0.20 if the post contains a link), capped at 2,000,000 reads/month. Legacy Basic ($200/mo, ~10–15k reads/mo) and Pro ($5,000/mo, ~1M reads/mo) plans are closed to new signups and being force-migrated to pay-per-use (Basic subscribers migrated after June 1, 2026). Enterprise starts around $42,000/month. [postproxy.dev, 2026](https://postproxy.dev/blog/x-api-pricing-2026); [blotato.com, 2026](https://www.blotato.com/blog/twitter-api-pricing); [xpoz.ai, 2026](https://www.xpoz.ai/blog/guides/understanding-twitter-api-pricing-tiers-and-alternatives) -One source claims reading **your own** account data is discounted to ~$0.001/request ("the only zero-cost access is reading your own account data at $0.001 per request" — [xpoz.ai](https://www.xpoz.ai/blog/guides/understanding-twitter-api-pricing-tiers-and-alternatives)), but this figure appears in only one secondary source and could not be cross-verified against X's own developer pricing page in this pass — **flag as low-confidence, unverified**. +One source claims reading **your own** account data is discounted to ~$0.001/request ("the only zero-cost access is reading your own account data at $0.001 per request" - [xpoz.ai](https://www.xpoz.ai/blog/guides/understanding-twitter-api-pricing-tiers-and-alternatives)), but this figure appears in only one secondary source and could not be cross-verified against X's own developer pricing page in this pass - **flag as low-confidence, unverified**. **Bottom line for a self-service tool:** there is no meaningful free official API path in 2026. The data archive (§1a) plus manual/Premium analytics CSV (§1b) is the realistic zero-cost route for an individual reading their own tweets and engagement. @@ -61,11 +68,11 @@ One source claims reading **your own** account data is discounted to ~$0.001/req **UI path:** Profile picture → Settings → **Security and apps** tab → **Download your information** → confirm → Export. [help.medium.com](https://help.medium.com/hc/en-us/articles/115004745787-Export-your-account-data) (fetched via search cache; direct fetch returned HTTP 403 in this session, so treat as medium-confidence secondary confirmation of the official article's content, corroborated by three independent how-to posts below) -**Delivery:** A download link is emailed when the export is ready (processing time not specified in the sources found — **unverified**, budget for it being non-instant like other platforms' exports). +**Delivery:** A download link is emailed when the export is ready (processing time not specified in the sources found - **unverified**, budget for it being non-instant like other platforms' exports). -**Format:** ZIP archive containing multiple **HTML** files/folders (a "posts" folder with your own published articles, plus everything you've written including comments, listed reverse-chronologically). This is explicitly HTML, not Markdown or JSON — several third-party tools exist specifically to convert this export to Markdown (e.g. [gautamdhameja/medium-2-md](https://github.com/gautamdhameja/medium-2-md)). [jhsu98.medium.com](https://jhsu98.medium.com/how-to-export-all-your-medium-content-bb3dc7587a5a); [elye-project.medium.com](https://elye-project.medium.com/the-hidden-important-medium-writer-feature-zip-your-medium-data-5319a1b7b542) +**Format:** ZIP archive containing multiple **HTML** files/folders (a "posts" folder with your own published articles, plus everything you've written including comments, listed reverse-chronologically). This is explicitly HTML, not Markdown or JSON - several third-party tools exist specifically to convert this export to Markdown (e.g. [gautamdhameja/medium-2-md](https://github.com/gautamdhameja/medium-2-md)). [jhsu98.medium.com](https://jhsu98.medium.com/how-to-export-all-your-medium-content-bb3dc7587a5a); [elye-project.medium.com](https://elye-project.medium.com/the-hidden-important-medium-writer-feature-zip-your-medium-data-5319a1b7b542) -**Stats/claps in the export:** **Not confirmed present.** No source in this research pass documents per-story view/read/clap counts inside the ZIP. The consistent description across sources is "posts as HTML" plus account/comment history — treat stats-in-export as **unverified, probably absent**. +**Stats/claps in the export:** **Not confirmed present.** No source in this research pass documents per-story view/read/clap counts inside the ZIP. The consistent description across sources is "posts as HTML" plus account/comment history - treat stats-in-export as **unverified, probably absent**. **On-screen stats dashboard:** Medium's built-in Stats page (accessible to any writer at `medium.com/me/stats`) shows per-story views, reads, read ratio, and fan/clap counts, but no official export/CSV button for it was found in this research pass. Third-party browser extensions exist specifically to scrape this screen (e.g. "Medium Stats Export" Chrome extension), which itself is evidence there's no native export. [Chrome Web Store listing](https://chromewebstore.google.com/detail/medium-stats-export/lnnogehgjdeccakdogbeinkhphconjbd?hl=en-GB) @@ -73,7 +80,7 @@ One source claims reading **your own** account data is discounted to ~$0.001/req Medium's public API is **deprecated**. Its GitHub repo is archived with a warning that it "is no longer supported and is not recommended for use." One source claims an integration-token feature is still technically reachable in account settings and unofficially functional, but this is unverified, unsupported, and should not be relied on for a public tool. [medium.surfingdoggo.com](https://medium.surfingdoggo.com/bummer-vote-with-a-comment-if-you-want-it-back-too-readers-unite-40a2c8b3a37a) (secondary/blog source; no official Medium statement located confirming current API status beyond the archived-repo notice) -**Bottom line:** For Medium, an open-source tool should plan on (1) parsing the HTML export for post text/dates/URLs, and (2) telling users that per-post stats have no export path — only the on-screen dashboard, scraped at their own risk (fragile, ToS-adjacent). +**Bottom line:** For Medium, an open-source tool should plan on (1) parsing the HTML export for post text/dates/URLs, and (2) telling users that per-post stats have no export path - only the on-screen dashboard, scraped at their own risk (fragile, ToS-adjacent). --- @@ -84,44 +91,44 @@ Medium's public API is **deprecated**. Its GitHub repo is archived with a warnin **UI path:** Settings → **Import/Export** (left sidebar, near the bottom of Settings) → **Export your data** → New export. [YouTube walkthrough transcript, 2026](https://www.youtube.com/watch?v=EsEg1JA5hnY); corroborated by [WebSearch summary of Substack support content, 2026] **Format:** A ZIP containing: -- `posts.csv` — a CSV listing of your posts +- `posts.csv` - a CSV listing of your posts - a `posts/` folder with **one HTML file per published post** - an email/subscriber list CSV (`email_list.[name].csv` per [Buttondown's Substack-import docs](https://docs.buttondown.com/substack)) -(Per WebSearch synthesis of Substack help content — not independently fetched from support.substack.com in this session due to a 403 on direct WebFetch; corroborated by the Buttondown migration doc which references the same export ZIP structure.) +(Per WebSearch synthesis of Substack help content - not independently fetched from support.substack.com in this session due to a 403 on direct WebFetch; corroborated by the Buttondown migration doc which references the same export ZIP structure.) **Subscriber export separately:** Subscribers page → **Export CSV** button (top right) → choose "all columns" (includes last-open date, individual post view data) or "visible columns only" (subscription date, plan type). [tella.com/Substack docs summary](https://www.tella.com/definition/subscriber-export) -### 3b. Per-post stats — export vs. dashboard only +### 3b. Per-post stats - export vs. dashboard only -The **Stats → Posts** page on Substack shows, per post: title, send date, audience, email deliveries and opens, open rate, free/paid subscriptions driven, likes, comments, shares — as an **on-screen sortable table**. [support.substack.com — A guide to Substack metrics](https://support.substack.com/hc/en-us/articles/5320347155860-A-guide-to-Substack-metrics) (direct fetch 403'd; content confirmed via Tavily search snippet of the same URL) +The **Stats → Posts** page on Substack shows, per post: title, send date, audience, email deliveries and opens, open rate, free/paid subscriptions driven, likes, comments, shares - as an **on-screen sortable table**. [support.substack.com - A guide to Substack metrics](https://support.substack.com/hc/en-us/articles/5320347155860-A-guide-to-Substack-metrics) (direct fetch 403'd; content confirmed via Tavily search snippet of the same URL) -Whether this table has a **native CSV export button** could not be confirmed from official docs in this pass — no source explicitly showed an "export" control on the Stats → Posts screen. The "export your data" flow in §3a (posts.csv) is a **content** export, not confirmed to include the same delivery/open/click numbers. +Whether this table has a **native CSV export button** could not be confirmed from official docs in this pass - no source explicitly showed an "export" control on the Stats → Posts screen. The "export your data" flow in §3a (posts.csv) is a **content** export, not confirmed to include the same delivery/open/click numbers. ### 3c. Substack's own analytics API (unofficial) Substack does not publish a public API, but its **internal REST endpoints** (used by its own dashboard) are reachable while authenticated as the account owner and have been reverse-engineered by third parties, e.g.: -- `/api/v1/post_management/detail/{post_id}` — per-post clicks (broken down by link), shares, likes, comments, restacks, signups/paid-conversions attributed to that post -- `/api/v1/publication/stats/email_stats` and `/email_stats/30d_open_rate` — per-post email delivery/open/click metrics -- `/api/v1/publication/stats/growth/sources` — subscriber source breakdown +- `/api/v1/post_management/detail/{post_id}` - per-post clicks (broken down by link), shares, likes, comments, restacks, signups/paid-conversions attributed to that post +- `/api/v1/publication/stats/email_stats` and `/email_stats/30d_open_rate` - per-post email delivery/open/click metrics +- `/api/v1/publication/stats/growth/sources` - subscriber source breakdown -[wonderingaboutai.substack.com, 2026](https://wonderingaboutai.substack.com/p/i-built-a-chrome-extension-to-audit) — **this is a single unofficial source (a hobbyist's write-up), not documented by Substack itself.** Flag as low-confidence / unofficial; these are not public API contracts and could change or be blocked without notice. The same source notes a parsing gotcha: Substack's "clicks" metric folds in clicks on the in-email "Like" button, inflating click-through-rate unless explicitly filtered out. +[wonderingaboutai.substack.com, 2026](https://wonderingaboutai.substack.com/p/i-built-a-chrome-extension-to-audit) - **this is a single unofficial source (a hobbyist's write-up), not documented by Substack itself.** Flag as low-confidence / unofficial; these are not public API contracts and could change or be blocked without notice. The same source notes a parsing gotcha: Substack's "clicks" metric folds in clicks on the in-email "Like" button, inflating click-through-rate unless explicitly filtered out. -**Bottom line:** Substack gives authors a real content export (posts.csv + HTML + subscriber CSV) but the most granular per-post engagement numbers (opens, click detail) live primarily in the on-screen Stats dashboard and an unofficial, unauthenticated-by-Substack internal API — there is no confirmed official, documented per-post-stats export file. +**Bottom line:** Substack gives authors a real content export (posts.csv + HTML + subscriber CSV) but the most granular per-post engagement numbers (opens, click detail) live primarily in the on-screen Stats dashboard and an unofficial, unauthenticated-by-Substack internal API - there is no confirmed official, documented per-post-stats export file. --- ## 4. YouTube -### 4a. YouTube Studio Analytics — CSV/Sheets export +### 4a. YouTube Studio Analytics - CSV/Sheets export -**UI path:** YouTube Studio → Analytics (left menu) → find the report → **Advanced Mode** (or "See more") → adjust as needed → **Export current view** → choose format. [Google Help — Advanced mode for analytics reports](https://support.google.com/youtube/answer/9717005?hl=en) +**UI path:** YouTube Studio → Analytics (left menu) → find the report → **Advanced Mode** (or "See more") → adjust as needed → **Export current view** → choose format. [Google Help - Advanced mode for analytics reports](https://support.google.com/youtube/answer/9717005?hl=en) **Cap:** Native Studio export via Advanced Mode is capped at **500 rows**; to get more than 500 rows of data you must use the **YouTube Reporting API** (bulk CSVs) instead. [Google Help, same page](https://support.google.com/youtube/answer/9717005?hl=en) **Per-video metrics available:** views, watch time (minutes), average view duration, likes, comments, shares, subscribers gained/lost, traffic-source breakdown (search, suggested, browse, external, playlists), audience demographics (age, gender, geography, device, playback location), and for monetized channels: estimated revenue, RPM, CPM. Date range customizable from 1 day to lifetime. [youtubetoolkit.com, 2026-06-25](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) (secondary source; cross-checked against the general shape of the official Analytics API metrics list) -**Data finalization delay:** Metrics are reported to finalize 48–72 hours after the fact; Studio shows provisional/estimated numbers before that. [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) — this specific delay window is from a secondary source, not an official Google statement located in this pass. +**Data finalization delay:** Metrics are reported to finalize 48–72 hours after the fact; Studio shows provisional/estimated numbers before that. [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) - this specific delay window is from a secondary source, not an official Google statement located in this pass. **500-video ceiling gotcha:** For channels with >500 videos, the native CSV export silently truncates with no warning; workaround is manual date-range slicing and merging, or using the API. [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) @@ -129,23 +136,23 @@ Substack does not publish a public API, but its **internal REST endpoints** (use Two distinct official APIs, both OAuth 2.0-only for private per-channel data: -- **YouTube Analytics API** (`reports.query`) — targeted queries, `startDate`/`endDate`/`metrics`/`dimensions`/`filters` parameters, JSON or CSV response. Quota: **10,000 queries/day**, no paid tier to buy more — you must submit a manual quota-increase audit form to Google and wait. [Google Developers — Reports: Query](https://developers.google.com/youtube/analytics/reference/reports/query); [youtubetoolkit.com](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) - - Traffic-source report constraint: `(# videos queried) × (# days in range) ≤ 50,000` or the API errors. [Google Developers — Revision History](https://developers.google.com/youtube/reporting/revision_history) -- **YouTube Reporting API** — bulk daily CSV reports (per-24-hour period), designed for large datasets; reports containing non-historical data available for **60 days**, historical-backfill reports available for **30 days**, after which they become inaccessible. [Google Developers — YouTube Reporting API](https://developers.google.com/youtube/reporting/v1/reports) +- **YouTube Analytics API** (`reports.query`) - targeted queries, `startDate`/`endDate`/`metrics`/`dimensions`/`filters` parameters, JSON or CSV response. Quota: **10,000 queries/day**, no paid tier to buy more - you must submit a manual quota-increase audit form to Google and wait. [Google Developers - Reports: Query](https://developers.google.com/youtube/analytics/reference/reports/query); [youtubetoolkit.com](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) + - Traffic-source report constraint: `(# videos queried) × (# days in range) ≤ 50,000` or the API errors. [Google Developers - Revision History](https://developers.google.com/youtube/reporting/revision_history) +- **YouTube Reporting API** - bulk daily CSV reports (per-24-hour period), designed for large datasets; reports containing non-historical data available for **60 days**, historical-backfill reports available for **30 days**, after which they become inaccessible. [Google Developers - YouTube Reporting API](https://developers.google.com/youtube/reporting/v1/reports) -**YouTube Data API v3** (separate from Analytics) — public metadata (titles, descriptions, tags, public view/like/comment counts) via API key, or OAuth for private data. Quota: 10,000 units/day; `playlistItems.list` and `videos.list` cost 1 unit per call (50 items), `search.list` costs 100 units/call — guidance is to avoid `search.list` and instead page through `playlistItems.list` → `videos.list` in batches of 50, which keeps even a 5,000-video channel under 50 units/day. [Google Developers — YouTube Data API v3](https://developers.google.com/youtube/v3); cost breakdown per [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) and [Bin-Huang/youtube-analytics-cli](https://github.com/Bin-Huang/youtube-analytics-cli) +**YouTube Data API v3** (separate from Analytics) - public metadata (titles, descriptions, tags, public view/like/comment counts) via API key, or OAuth for private data. Quota: 10,000 units/day; `playlistItems.list` and `videos.list` cost 1 unit per call (50 items), `search.list` costs 100 units/call - guidance is to avoid `search.list` and instead page through `playlistItems.list` → `videos.list` in batches of 50, which keeps even a 5,000-video channel under 50 units/day. [Google Developers - YouTube Data API v3](https://developers.google.com/youtube/v3); cost breakdown per [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) and [Bin-Huang/youtube-analytics-cli](https://github.com/Bin-Huang/youtube-analytics-cli) -Both official APIs are **free of monetary charge** — the constraint is the daily quota, not billing. [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) +Both official APIs are **free of monetary charge** - the constraint is the daily quota, not billing. [youtubetoolkit.com, 2026](https://youtubetoolkit.com/blog/how-to-get-my-youtube-channel-data-as-csv) ### 4c. Google Takeout for YouTube **UI path:** `takeout.google.com` → Deselect all → select **YouTube and YouTube Music** → choose categories (playlists, subscriptions, comments/live chat, history, videos) → choose format (JSON recommended over legacy HTML for `history`; **playlists, subscriptions, and comments always export as CSV with no format choice**) → Create export. [quiki.io, 2026](https://quiki.io/how-to-download-youtube-data); confirmed structurally by [Google's Data Portability API schema reference](https://developers.google.com/data-portability/schema-reference/youtube) -**What it contains:** video files (original format if uploaded <~6 months ago, otherwise transcoded MP4/H.264+AAC), playlists (CSV: one file per playlist, video ID + date-added only, not full metadata), subscriptions, your own comments/live chats, liked videos, channel posts. **Does not include per-video analytics/engagement** — Takeout is a content/activity backup, not an analytics export. [tubefetcher.com, 2026-03-22](https://tubefetcher.com/blog/archive-youtube-channel-backup/); [karlicoss/google_takeout_parser](https://github.com/karlicoss/google_takeout_parser) +**What it contains:** video files (original format if uploaded <~6 months ago, otherwise transcoded MP4/H.264+AAC), playlists (CSV: one file per playlist, video ID + date-added only, not full metadata), subscriptions, your own comments/live chats, liked videos, channel posts. **Does not include per-video analytics/engagement** - Takeout is a content/activity backup, not an analytics export. [tubefetcher.com, 2026-03-22](https://tubefetcher.com/blog/archive-youtube-channel-backup/); [karlicoss/google_takeout_parser](https://github.com/karlicoss/google_takeout_parser) **Parser gotchas:** -- Comment CSVs store the actual comment text as a **nested JSON blob** inside a CSV cell (a `takeoutSegments` array with text/link fragments), not as plain text — requires a second parse pass. [karlicoss/google_takeout_parser README](https://github.com/karlicoss/google_takeout_parser) -- Video filenames in the export are **truncated to 55 characters** and de-duplicated with `(1)`, `(2)` suffixes, with **no reliable field linking the truncated filename back to the video ID** in the accompanying metadata CSVs — a documented unsolved matching problem. [Stack Overflow: Working with Google Takeout's video metadata](https://stackoverflow.com/questions/79277419/working-with-google-takeouts-video-metadata) +- Comment CSVs store the actual comment text as a **nested JSON blob** inside a CSV cell (a `takeoutSegments` array with text/link fragments), not as plain text - requires a second parse pass. [karlicoss/google_takeout_parser README](https://github.com/karlicoss/google_takeout_parser) +- Video filenames in the export are **truncated to 55 characters** and de-duplicated with `(1)`, `(2)` suffixes, with **no reliable field linking the truncated filename back to the video ID** in the accompanying metadata CSVs - a documented unsolved matching problem. [Stack Overflow: Working with Google Takeout's video metadata](https://stackoverflow.com/questions/79277419/working-with-google-takeouts-video-metadata) - Download links expire after about a week; Takeout is a static one-time snapshot with no incremental/delta re-export. [tubefetcher.com, 2026](https://tubefetcher.com/blog/archive-youtube-channel-backup/) --- @@ -154,19 +161,19 @@ Both official APIs are **free of monetary charge** — the constraint is the dai ### 5a. GDPR/CCPA-style data request -**UI path:** `reddit.com/settings/data-request` (must be logged into the account requesting data) → follow prompts → Submit. [Reddit Help — How do I request a copy of my Reddit data and information?](https://support.reddithelp.com/hc/en-us/articles/360043048352-How-do-I-request-a-copy-of-my-Reddit-data-and-information) +**UI path:** `reddit.com/settings/data-request` (must be logged into the account requesting data) → follow prompts → Submit. [Reddit Help - How do I request a copy of my Reddit data and information?](https://support.reddithelp.com/hc/en-us/articles/360043048352-How-do-I-request-a-copy-of-my-Reddit-data-and-information) **Turnaround:** Up to **30 days** to prepare; Reddit sends a notification to the account's Reddit inbox with a download link when ready. [Reddit Help, same article](https://support.reddithelp.com/hc/en-us/articles/360043048352-How-do-I-request-a-copy-of-my-Reddit-data-and-information) -**Format:** Community/tooling reports describe the export as containing **`posts.csv`** and **`comments.csv`** among other files. [WebSearch summary citing Reddit GDPR export structure, 2026] — **this specific filename pair is from secondary/community sources (e.g. the "Reddit GDPR Export Data Viewer" tool and community guides), not an official Reddit schema document**, so treat exact filenames as likely-but-unconfirmed. Reddit's official help article does not appear to enumerate file names. +**Format:** Community/tooling reports describe the export as containing **`posts.csv`** and **`comments.csv`** among other files. [WebSearch summary citing Reddit GDPR export structure, 2026] - **this specific filename pair is from secondary/community sources (e.g. the "Reddit GDPR Export Data Viewer" tool and community guides), not an official Reddit schema document**, so treat exact filenames as likely-but-unconfirmed. Reddit's official help article does not appear to enumerate file names. -**Karma/score per post:** Not confirmed to be included in the GDPR export CSVs in any source found in this pass. Reddit's UI-accessible account settings (not the formal data request) separately expose posts, comments, votes, recent IPs, and preferences, per the same Reddit Help article — but again without a documented schema. +**Karma/score per post:** Not confirmed to be included in the GDPR export CSVs in any source found in this pass. Reddit's UI-accessible account settings (not the formal data request) separately expose posts, comments, votes, recent IPs, and preferences, per the same Reddit Help article - but again without a documented schema. ### 5b. Reddit API / PRAW for a user's own posts -PRAW (Python Reddit API Wrapper) is the standard client library; it supports pulling your own submissions via the Reddit API with fields like `id`, `title`, `url`, `permalink`, `subreddit`, `created_utc`, `score` (karma for that post), `num_comments`, etc. — standard Submission object attributes exposed by PRAW/Reddit's API. [danielkliewer.com, 2025-10-21](https://www.danielkliewer.com/blog/2025-10-21-ultimate-guide-export-your-reddit-data-to-markdown-using-python-and-PRAW-API); [karlicoss/rexport](https://github.com/karlicoss/rexport) (JSON export tool built on the same API surface: comments, submissions, upvotes) +PRAW (Python Reddit API Wrapper) is the standard client library; it supports pulling your own submissions via the Reddit API with fields like `id`, `title`, `url`, `permalink`, `subreddit`, `created_utc`, `score` (karma for that post), `num_comments`, etc. - standard Submission object attributes exposed by PRAW/Reddit's API. [danielkliewer.com, 2025-10-21](https://www.danielkliewer.com/blog/2025-10-21-ultimate-guide-export-your-reddit-data-to-markdown-using-python-and-PRAW-API); [karlicoss/rexport](https://github.com/karlicoss/rexport) (JSON export tool built on the same API surface: comments, submissions, upvotes) -**Rate/volume limit:** Reddit's API listing endpoints are commonly reported to cap at **1,000 items** per listing (a long-standing Reddit API constraint, not specific to 2026), which matters for accounts with very long posting histories. [WebSearch summary, 2026] — this is a widely repeated community claim; no official 2026 Reddit API doc was fetched in this pass to pin the exact current number, so treat as **likely correct but not freshly confirmed against Reddit's current developer docs**. +**Rate/volume limit:** Reddit's API listing endpoints are commonly reported to cap at **1,000 items** per listing (a long-standing Reddit API constraint, not specific to 2026), which matters for accounts with very long posting histories. [WebSearch summary, 2026] - this is a widely repeated community claim; no official 2026 Reddit API doc was fetched in this pass to pin the exact current number, so treat as **likely correct but not freshly confirmed against Reddit's current developer docs**. **Auth:** OAuth2 (script-type app registered at reddit.com/prefs/apps) with username/password or refresh-token flow for reading your own account's private-ish data (saved posts, vote history); public posts/comments are readable without auth for basic listing endpoints. @@ -176,35 +183,35 @@ PRAW (Python Reddit API Wrapper) is the standard client library; it supports pul ### 6a. Meta "Download your information" (Accounts Center) -**UI path:** Instagram Settings → **Accounts Center** → **Your information and permissions** → **Download your information** → Create export → select profile → choose destination (device or transfer to a linked service) → choose format (**JSON or HTML**) and media quality → Confirm. [Facebook Help — Access and download your information on Instagram](https://www.facebook.com/help/instagram/181231772500920) — official Meta help page, directly corroborates the step sequence found in secondary walkthroughs. +**UI path:** Instagram Settings → **Accounts Center** → **Your information and permissions** → **Download your information** → Create export → select profile → choose destination (device or transfer to a linked service) → choose format (**JSON or HTML**) and media quality → Confirm. [Facebook Help - Access and download your information on Instagram](https://www.facebook.com/help/instagram/181231772500920) - official Meta help page, directly corroborates the step sequence found in secondary walkthroughs. -**Turnaround:** Meta's official stated ceiling is **up to 30 days**; a narrow request (e.g. only "Saved") is anecdotally reported to often complete within hours to a couple of days. [redact.dev](https://redact.dev/blog/instagram-data-file-download) (secondary source for the "hours to days in practice" claim — the 30-day figure is Meta's own official SLA language, seen consistently across Meta help content). +**Turnaround:** Meta's official stated ceiling is **up to 30 days**; a narrow request (e.g. only "Saved") is anecdotally reported to often complete within hours to a couple of days. [redact.dev](https://redact.dev/blog/instagram-data-file-download) (secondary source for the "hours to days in practice" claim - the 30-day figure is Meta's own official SLA language, seen consistently across Meta help content). **Format contents:** JSON gives structured data per category (posts, comments, follows, messages, ads activity, account history); HTML gives a clickable browsable version of the same. A `saved_posts` file/category provides only **links + timestamps** to saved content, not the saved posts' actual content. [WebSearch synthesis, 2026, based on multiple how-to guides] -**Critical gotcha — insights are NOT meaningfully in this export:** Per a source explicitly warning about this exact confusion: "You may find a 'Download your information' feature... Avoid this for analytics. That feature is designed for data privacy compliance... The insights data included there isn't organized for reporting." [trypostbase.com, 2025-10-31](https://www.trypostbase.com/resources/how-to-export-instagram-insights-data) — i.e., per-post reach/impressions/saves are not usably present in the Accounts Center export; you must use Insights (§6b) or the Graph API (§6c). +**Critical gotcha - insights are NOT meaningfully in this export:** Per a source explicitly warning about this exact confusion: "You may find a 'Download your information' feature... Avoid this for analytics. That feature is designed for data privacy compliance... The insights data included there isn't organized for reporting." [trypostbase.com, 2025-10-31](https://www.trypostbase.com/resources/how-to-export-instagram-insights-data) - i.e., per-post reach/impressions/saves are not usably present in the Accounts Center export; you must use Insights (§6b) or the Graph API (§6c). ### 6b. Instagram/Threads Professional dashboard & Insights (on-screen) -**Requires a Professional (Business or Creator) account** — personal accounts have no Insights at all. [help.instagram.com — About Instagram insights](https://help.instagram.com/788388387972460/) +**Requires a Professional (Business or Creator) account** - personal accounts have no Insights at all. [help.instagram.com - About Instagram insights](https://help.instagram.com/788388387972460/) -**Per-post/reel/story metrics on-screen:** views, reach ("Viewers" — unique accounts, distinct from views which counts repeats), likes, comments, saves, shares, profile activity/visits, follows generated, and for Reels: plays, watch time, skip rate. Account-level Insights covers reach, accounts engaged, follower demographics, profile activity, over 7/14/30/90-day windows — capped at roughly **90 days of history** on-screen. [postzen.dev, 2026-09-09](https://www.postzen.dev/blog/instagram-insights-explained); [inro.social, 2026-01-28](https://www.inro.social/blog/instagram-dashboard-professional-dashboard-guide); [help.instagram.com](https://help.instagram.com/788388387972460/) +**Per-post/reel/story metrics on-screen:** views, reach ("Viewers" - unique accounts, distinct from views which counts repeats), likes, comments, saves, shares, profile activity/visits, follows generated, and for Reels: plays, watch time, skip rate. Account-level Insights covers reach, accounts engaged, follower demographics, profile activity, over 7/14/30/90-day windows - capped at roughly **90 days of history** on-screen. [postzen.dev, 2026-09-09](https://www.postzen.dev/blog/instagram-insights-explained); [inro.social, 2026-01-28](https://www.inro.social/blog/instagram-dashboard-professional-dashboard-guide); [help.instagram.com](https://help.instagram.com/788388387972460/) **CSV export path for Insights (the one that actually works):** Instagram → More → **Meta Business Suite** → Insights → Content tab → **Export Data** (top right) → pick account/date-range/post-or-story → Generate → Download. This produces a real CSV (not PDF). [poststeady.com, 2026-08-31](https://www.poststeady.com/resources/export-instagram-insights-to-csv); [trypostbase.com, 2025-10-31](https://www.trypostbase.com/resources/how-to-export-instagram-insights-data) -**Column-naming gotcha (breaks naive parsers):** Meta Business Suite's exported CSV headers do **not** match the on-screen Insights labels. E.g. there is no column literally called "Reach" — it's "Accounts reached"; "Likes" may appear as "Reactions"; "Shares" may appear as "Reposts", depending on which export variant was generated — and these header names have been observed to shift between export runs. [poststeady.com, 2026-08-31](https://www.poststeady.com/resources/export-instagram-insights-to-csv) +**Column-naming gotcha (breaks naive parsers):** Meta Business Suite's exported CSV headers do **not** match the on-screen Insights labels. E.g. there is no column literally called "Reach" - it's "Accounts reached"; "Likes" may appear as "Reactions"; "Shares" may appear as "Reposts", depending on which export variant was generated - and these header names have been observed to shift between export runs. [poststeady.com, 2026-08-31](https://www.poststeady.com/resources/export-instagram-insights-to-csv) ### 6c. Instagram Graph API (official, for developers) -Endpoints: `GET //insights` (per-post) and `GET //insights` (account-level), OAuth-authenticated, only works for **professional accounts** (Business/Creator) — cannot fetch data for personal-account media at all. [developers.facebook.com — Instagram Platform Insights](https://developers.facebook.com/docs/instagram-platform/insights/) +Endpoints: `GET //insights` (per-post) and `GET //insights` (account-level), OAuth-authenticated, only works for **professional accounts** (Business/Creator) - cannot fetch data for personal-account media at all. [developers.facebook.com - Instagram Platform Insights](https://developers.facebook.com/docs/instagram-platform/insights/) **Metric naming churn (gotcha for a parser/tool):** `impressions` was **removed from the API on April 21, 2025**, replaced by `views` as the primary metric; `saved` (not "saves") is the correct API field name; `reach` is explicitly documented as "**estimated**", not exact. New metrics like `reposts` (Dec 2025) and story `link_clicks` (June 2026) have been added recently, meaning a metric list written even a year prior would already be stale. [postzen.dev, 2026-09-09](https://www.postzen.dev/blog/instagram-insights-explained), which itself cross-cites the official [developers.facebook.com Instagram Media Insights reference](https://developers.facebook.com/docs/instagram-platform/reference/instagram-media/insights/). -**Auth difficulty:** Requires a registered Meta developer app, Instagram professional-account login/business-asset linkage, permission review for some scopes, and token refresh handling — materially harder to bootstrap solo than X's or YouTube's OAuth flows, though the API itself is free of direct monetary cost. +**Auth difficulty:** Requires a registered Meta developer app, Instagram professional-account login/business-asset linkage, permission review for some scopes, and token refresh handling - materially harder to bootstrap solo than X's or YouTube's OAuth flows, though the API itself is free of direct monetary cost. ### 6d. Threads specifically -Threads has its own "Threads Insights" (AI-summarized as of an August 2026 Meta update) but this research pass did not locate a confirmed, separate Threads-only export or API distinct from the general Meta Accounts Center "Download your information" flow and the Instagram/Threads shared Graph API surface. [embedsocial.com, 2026](https://embedsocial.com/blog/new-threads-features-2026) mentions the Insights AI-summary feature but not an export mechanism — **flag Threads-specific export/API details as not independently verified in this pass; likely follows the same Accounts Center path as Instagram since both sit under one Meta account, but this should be confirmed directly against a live account before shipping a parser.** +Threads has its own "Threads Insights" (AI-summarized as of an August 2026 Meta update) but this research pass did not locate a confirmed, separate Threads-only export or API distinct from the general Meta Accounts Center "Download your information" flow and the Instagram/Threads shared Graph API surface. [embedsocial.com, 2026](https://embedsocial.com/blog/new-threads-features-2026) mentions the Insights AI-summary feature but not an export mechanism - **flag Threads-specific export/API details as not independently verified in this pass; likely follows the same Accounts Center path as Instagram since both sit under one Meta account, but this should be confirmed directly against a live account before shipping a parser.** --- @@ -215,16 +222,16 @@ Threads has its own "Threads Insights" (AI-summarized as of an August 2026 Meta | X/Twitter | Yes (ZIP, JSON-in-.js) | No (counts present but reported non-real/zeroed) | Yes, CSV via analytics.x.com | Yes (Premium, ~$8/mo) for dashboard+export; free per-post view is view-only, no export | Pay-per-use only in 2026, no free tier | | Medium | Yes (ZIP of HTML) | Not confirmed present | No confirmed export; dashboard only | No (stats page is free) but no export | API deprecated/unsupported | | Substack | Yes (posts.csv + HTML + subscriber CSV) | Not confirmed in that export | Stats→Posts is on-screen; no confirmed CSV button; unofficial internal API exists | No (native features free) | No public API; unofficial internal endpoints only | -| YouTube | Yes (Takeout, CSV/JSON/video files) | No (Takeout has no analytics) | Yes — Studio CSV (500-row cap) or Reporting/Analytics API (bulk) | No (all free; quota-limited not paywalled) | Yes, free, quota-capped (10,000 units/day) | +| YouTube | Yes (Takeout, CSV/JSON/video files) | No (Takeout has no analytics) | Yes - Studio CSV (500-row cap) or Reporting/Analytics API (bulk) | No (all free; quota-limited not paywalled) | Yes, free, quota-capped (10,000 units/day) | | Reddit | Yes (GDPR request, ~30 days) | Possibly (community-reported posts.csv/comments.csv, unconfirmed schema) | Unclear/unconfirmed for karma-per-post in official export | No | Yes, via PRAW/OAuth, ~1,000-item listing cap (unconfirmed current figure) | -| Instagram/Threads | Yes (Accounts Center, JSON/HTML) | No — explicitly not analytics-grade | Yes — Meta Business Suite CSV export | No (Insights free for Professional accounts) | Yes, Graph API, professional accounts only, metric names churn frequently | +| Instagram/Threads | Yes (Accounts Center, JSON/HTML) | No - explicitly not analytics-grade | Yes - Meta Business Suite CSV export | No (Insights free for Professional accounts) | Yes, Graph API, professional accounts only, metric names churn frequently | --- ## Notes on Research Confidence - **High confidence / directly corroborated by an official platform help page:** X archive format and contents (§1a), Reddit's GDPR request path and 30-day SLA (§5a), YouTube Studio export path and 500-row cap (§4a), YouTube API quota mechanics (§4b), Meta's Download Your Information UI path (§6a), Instagram Insights account-type requirement (§6b). -- **Medium confidence / consistent across several independent secondary sources but not fetched from the platform's own docs in this session:** Medium's export path/contents (§2a — help.medium.com blocked direct fetch, relied on search-cache + 3 independent how-to posts), Substack's export structure (§3a — support.substack.com blocked direct fetch), X Analytics CSV column list (§1b). +- **Medium confidence / consistent across several independent secondary sources but not fetched from the platform's own docs in this session:** Medium's export path/contents (§2a - help.medium.com blocked direct fetch, relied on search-cache + 3 independent how-to posts), Substack's export structure (§3a - support.substack.com blocked direct fetch), X Analytics CSV column list (§1b). - **Low confidence / single-source or explicitly unofficial:** Substack's internal stats API endpoints (§3c, one hobbyist blog), Reddit GDPR export filenames (§5a, community tooling only), the claimed $0.001/request "own tweets" discount on the X API (§1c, one source), Threads-specific export mechanics (§6d, not directly found). - **Not found / explicitly flagged as gaps:** an official, documented schema for Reddit's GDPR export files; a native CSV-export button on Substack's Stats→Posts screen; confirmation of Medium export processing time; any evidence engagement counts in the X archive are real rather than placeholder zeros. diff --git a/skills/jev-writer/references/providers.md b/skills/jev-writer/references/providers.md index a21300d..4faea29 100644 --- a/skills/jev-writer/references/providers.md +++ b/skills/jev-writer/references/providers.md @@ -4,8 +4,14 @@ Jev is reachable through three transports today. The tool auto-detects whichever key is present, so the user sets one environment variable and nothing else. **Never ask a user to paste an API key into a chat.** Have them set it in their -shell or a local `.env`, then run `jev-writer doctor`, which confirms the key -works without ever displaying it. +shell or a local `.env`, then run `doctor`, which confirms the key works without +ever displaying it: + +```bash +npx github:Kaos599/jev-writer doctor # or, from a clone: node src/cli.mjs doctor +``` + +The package is not on npm, so `npx jev-writer doctor` does not resolve. ## The three routes @@ -18,6 +24,9 @@ works without ever displaying it. Recommend whichever the user already has an account with. If they have none, OpenRouter is usually the fastest to obtain. +Set exactly one. If several are set, the first present in the order above wins, +and `doctor` prints both which one it chose and a note that others were ignored. + ## Why these are not interchangeable URLs They are three different protocols, which is why each gets its own adapter file @@ -41,8 +50,8 @@ provider translates on the way in and out. ## Cost Jev is priced at **$0.042 per million input tokens, output free**. Rating 178 -posts with 30 questions at 5 repetitions each consumed roughly 7.4M input -tokens — about $0.31. +posts with 30 questions at 5 repetitions each is 890 calls, which consumed +roughly 7.4M input tokens: about $0.31. The practical consequence is methodological rather than financial: because a full pass is nearly free, rubrics can be rewritten and re-run many times while @@ -55,11 +64,16 @@ September 2026 and pricing has been moving. This tool sends a person's private writing to a third party. Treat that as a material fact, not a footnote. -- Zero Data Retention is requested on every AI Gateway call. +- Zero Data Retention is requested on every AI Gateway call + (`providerOptions.gateway.zeroDataRetention`). The other two transports have no + equivalent flag in this codebase, so on those routes the provider's own default + applies. - Tell the user which provider their content will reach, and point them at that provider's terms rather than making guarantees on the provider's behalf. - Never commit a corpus, ratings, or a generated dashboard that embeds post text. - The repo's `.gitignore` excludes `data/`, `out/` and `*.jsonl` for this reason. + The repo's `.gitignore` excludes the whole output directory (`jev-out/`) plus + `data/`, `out/`, `*.jsonl`, `dashboard.html` and `report.json` for this reason. + An earlier version listed only file types and missed `jev-out/report.json`. ## Adding a fourth provider diff --git a/skills/jev-writer/references/rubric-authoring.md b/skills/jev-writer/references/rubric-authoring.md index 18094d6..a5605aa 100644 --- a/skills/jev-writer/references/rubric-authoring.md +++ b/skills/jev-writer/references/rubric-authoring.md @@ -4,6 +4,14 @@ A pack is a versioned set of typed questions plus the state fields each question is allowed to see. The LinkedIn pack ships in the box; packs for newsletters, video titles, documentation or cold email are ordinary user-space files. +The format lives in `src/rubrics/pack.mjs` in the repo. From outside the package +it is imported as `jev-writer/rubrics/pack.mjs`; deep paths into `src/` do not +resolve, because the `exports` map in `package.json` is the public surface. + +```js +import { score, choice, bool, validatePack } from 'jev-writer/rubrics/pack.mjs'; +``` + ## The three primitives Choose by what the answer *means*, not by what is convenient. @@ -15,45 +23,80 @@ Choose by what the answer *means*, not by what is convenient. | Whether a condition holds | `boolean` | probability of true | A common authoring mistake is reaching for `choice` when the options are ordered. -Shelf life — ephemeral, timely, medium-lived, evergreen — is ordered, so it is a +Shelf life (ephemeral, timely, medium-lived, evergreen) is ordered, so it is a `score`. As a `choice` you throw away the ordering and get back a label you cannot correlate. `validatePack` cannot catch this; think about it. -## Hard constraints +Only `score` and `boolean` questions are correlated against outcomes, banded or +graded. `choice` answers are shown but never ranked, because a nominal label has +no direction to rank along. -- `score` takes **2 to 10 levels**. The API rejects anything else. -- `choice` needs at least two options and **must include a no-match outcome** - (`none_of_these`). Without one you force a label onto content that fits none of - them, which quietly manufactures data. `validatePack` enforces this. -- Every question needs a `tier` (`primary` or `exploratory`) and every rated - dimension needs a `direction`. +## What `validatePack` actually checks -## Pre-registration is not decoration +Run it before spending a single call. It returns an array of human-readable +problems, and an empty array means valid. It checks: -`tier` records a commitment made **before** any result was seen. Because packs -live in git, the commit timestamp is the evidence. +- the pack has an `id`, a `version`, a `description`, non-empty `stateFields` and + at least one question, +- every question id is snake_case, +- every `type` is one of `score`, `choice`, `boolean`, +- every `tier` is `primary` or `exploratory`, +- every question has `instructions`, +- every `score` has between 2 and 10 levels, +- every `choice` has at least two options and one of them is `none_of_these`, + `other` or `no_match`, +- every backticked reference inside an instruction names a field the pack + declared in `stateFields`. -Without it, scoring 30 dimensions against 2 outcomes and reporting whichever -three correlate is indistinguishable from making things up: at 60 tests, about -three will look significant on pure noise. Mark a question `primary` only when -you have a mechanism you would act on. Everything else is `exploratory`, which -means it will be computed, shown, and never called a finding. +**It does not check direction of merit.** See the next section. -Changing a tier after seeing results defeats the entire mechanism. Do not. +## Direction is required, but not by `validatePack` -## Direction is required +Direction of merit is a separate `directions` export beside the pack, mapping +each rated dimension to `higher_is_better`, `lower_is_better`, or `neutral`: -`higher_is_better`, `lower_is_better`, or `neutral`. +```js +export const directions = { + hook_strength: 'higher_is_better', + ai_generated_feel: 'lower_is_better', + technicality: 'neutral', +}; +``` High `human_authorship_evidence` is good; high `ai_generated_feel` is bad; `technicality` is neither, it is a choice about audience. Nothing in the question text says which, so without this metadata a colour scale paints the worst posts -green. The report layer throws rather than guessing. +green. + +Enforcement happens at runtime in the report layer, not at validation time. +`buildReport` throws when a rated dimension has no direction, and the CLI skips +report generation with a warning when a pack has no directions entry at all. So a +pack can pass `validatePack`, rate a whole corpus, and then produce no +`report.json` and no dashboard. Write the directions at the same time as the +questions. + +Only `score` and `boolean` questions need one. The shipped LinkedIn pack has 30 +questions, 24 of which are rated and therefore carry a direction: 13 +`higher_is_better`, 7 `lower_is_better`, 4 `neutral`. + +## Pre-registration is not decoration + +`tier` records a commitment made **before** any result was seen. Because packs +live in git, the commit timestamp is the evidence. + +Without it, scoring two dozen dimensions against two outcomes and reporting +whichever three correlate is indistinguishable from making things up: at 48 +tests, roughly 2.4 will look significant on pure noise. Mark a question `primary` +only when you have a mechanism you would act on. Everything else is +`exploratory`, which means it will be computed, shown, and never called a +finding. + +Changing a tier after seeing results defeats the entire mechanism. Do not. ## Never ask the model anything code can count Length, word count, emoji, hashtags, line breaks, list markers, posting hour, -days since the last post, media type — all computed exactly by the adapter, all +days since the last post, media type: all computed exactly by the adapter, all fed into the same analysis, all competing with the model's judgments on equal footing. @@ -64,7 +107,7 @@ the model for the count. ## Wording is load-bearing -`jev-1.13` reads instructions literally — it answers the question you wrote, not +`jev-1.13` reads instructions literally. It answers the question you wrote, not the one you meant. Measured during development: tightening one clause in a single boolean, from @@ -74,7 +117,7 @@ moved its probability from **0.99 to 0.71 on identical input**. Therefore: - Every `score` level is an object with a concrete `summary` and observable - `signals`, and must stand alone — each level is judged on its own. + `signals`, and must stand alone, because each level is judged on its own. - Every `choice` option carries `what`, `not_for` and `examples` to sharpen the boundary against its neighbours. - Put boundary cases in the criteria, not in your head. When you look at a wrong @@ -94,10 +137,22 @@ everything else. Two reasons this is enforced rather than trusted: - A rubric that can see the engagement it is meant to predict is not measuring anything. Outcome fields must never reach the model, and the tests assert it. +## Groups are presentation only + +A pack may declare a `groups` array, each entry naming a `key`, a `title` and the +dimensions it contains. The LinkedIn pack declares five families covering all 30 +questions. The dashboard uses them to organise the expanded post view, and +nothing else does: grouping never touches how a question is asked, rated or +correlated. A pack that declares none still renders, with every question in one +list, and anything a pack leaves unplaced falls into a catch-all. + ## Custom questions To add your own dimensions to an existing pack, copy it, bump the version, add -questions with their tier and direction, and run `validatePack` before spending a -single call. A future release will support a user config file for this so the -pack itself does not need forking; until then, a copied pack is the supported -path and is version-controlled, which is the property that matters. +questions with their tier, add their direction to the `directions` export, and +run `validatePack` before spending a single call. Then register the pack and its +directions in `src/cli.mjs` so the commands can find it. + +A future release will support a user config file so the pack itself does not need +forking; until then, a copied pack is the supported path and is version +controlled, which is the property that matters. diff --git a/src/adapters/linkedin.mjs b/src/adapters/linkedin.mjs index 38e8678..e775f7e 100644 --- a/src/adapters/linkedin.mjs +++ b/src/adapters/linkedin.mjs @@ -204,6 +204,43 @@ const parseUsDate = (s) => { return m ? `${m[3]}-${String(+m[1]).padStart(2, '0')}-${String(+m[2]).padStart(2, '0')}` : null; }; +// Shares.csv dates are ISO with a time part. Only the day is ever used. +const ISO_DATE = /^\d{4}-\d{2}-\d{2}/; + +// How many offending raw values to quote back. Enough to recognise a format +// change at a glance, few enough that a broken export does not fill the screen. +const DROP_SAMPLES = 3; + +const DROP_LABELS = { + post: 'post row(s) in Shares_*.csv', + media: 'row(s) in Rich_Media.csv', + metric: 'analytics row(s) under TOP POSTS', + follower: 'row(s) in the FOLLOWERS sheet', +}; + +// Tolerance on the follower back-walk consistency check, as a fraction of the +// headline total. The headline and the daily rows are computed at different +// moments, so an exact match is not expected; a larger disagreement means the +// two are not describing the same window. +const FOLLOWER_TOTAL_TOLERANCE = 0.005; + +// Rounding-safe ratio. A zero NUMERATOR is a real outcome and must survive; +// only a zero denominator is undefined. +const ratio = (num, den, digits) => + Number.isFinite(num) && Number.isFinite(den) && den !== 0 ? +(num / den).toFixed(digits) : null; + +/** One warning per source that lost rows, naming the count and the first few raw values. */ +function describeDrops(dropped, warnings) { + for (const [key, raws] of Object.entries(dropped)) { + if (!raws.length) continue; + const sample = raws.slice(0, DROP_SAMPLES).map((r) => JSON.stringify(r)).join(', '); + warnings.push( + `${raws.length} ${DROP_LABELS[key]} had an unreadable date and were skipped. ` + + `First value(s): ${sample}.`, + ); + } +} + function findFile(dir, re) { const hit = readdirSync(dir).find((f) => re.test(f)); return hit ? join(dir, hit) : null; @@ -211,6 +248,10 @@ function findFile(dir, re) { export function buildCorpus(rawDir, { sinceYears = null } = {}) { const warnings = []; + // Rows whose date will not parse used to vanish without a trace, which made a + // silently halved corpus look exactly like a small one. Collect them instead; + // a malformed export is a warning to the user, not a reason to throw. + const dropped = { post: [], media: [], metric: [], follower: [] }; const sharesPath = findFile(rawDir, /^Shares.*\.csv$/i); if (!sharesPath) { throw new Error( @@ -225,10 +266,17 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { if (!xlsxPath) warnings.push('No analytics .xlsx found: no engagement outcomes, so only descriptives are possible.'); // posts - let posts = parseCsv(readFileSync(sharesPath, 'utf8')) - .map((r) => ({ date: (r.Date ?? '').trim(), text: (r.ShareCommentary ?? '').trim(), share_link: r.ShareLink ?? '', visibility: r.Visibility ?? '' })) - .filter((p) => p.date && p.text) - .sort((a, b) => a.date.localeCompare(b.date)); + let posts = []; + for (const r of parseCsv(readFileSync(sharesPath, 'utf8'))) { + const date = (r.Date ?? '').trim(); + const text = (r.ShareCommentary ?? '').trim(); + // A reshare with no commentary carries no writing to rate; that is not a + // parse failure and does not deserve a warning. + if (!text) continue; + if (!ISO_DATE.test(date)) { dropped.post.push(date); continue; } + posts.push({ date, text, share_link: r.ShareLink ?? '', visibility: r.Visibility ?? '' }); + } + posts.sort((a, b) => a.date.localeCompare(b.date)); if (sinceYears) { const cut = new Date(Date.now() - sinceYears * 365.25 * 864e5).toISOString().slice(0, 10); @@ -241,10 +289,10 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { for (const r of parseCsv(readFileSync(richPath, 'utf8'))) { const raw = r['Date/Time'] ?? ''; const dm = RM_DATE.exec(raw); - if (!dm) continue; + if (!dm) { dropped.media.push(raw); continue; } const [, mon, day, yr] = /(\w+) (\d{1,2}), (\d{4})/.exec(dm[1]) ?? []; const mi = MONTHS.indexOf(mon); - if (mi < 0) continue; + if (mi < 0) { dropped.media.push(raw); continue; } const d = `${yr}-${String(mi + 1).padStart(2, '0')}-${String(+day).padStart(2, '0')}`; const kind = (RM_KIND.exec(raw) ?? [, 'unknown'])[1]; if (!media.has(d)) media.set(d, new Map()); @@ -262,37 +310,89 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { if (String(c[0]).startsWith('http')) { const d = parseUsDate(c[1]); if (d) metrics.set(d, { ...(metrics.get(d) ?? {}), post_url: c[0], engagements: Math.round(+c[2] || 0) }); + else dropped.metric.push(String(c[1])); } if (String(c[4]).startsWith('http')) { const d = parseUsDate(c[5]); if (d) metrics.set(d, { ...(metrics.get(d) ?? {}), post_url: c[4], impressions: Math.round(+c[6] || 0) }); + else dropped.metric.push(String(c[5])); } } + // Reconstruct follower count at any date by walking back from the total. + // + // The sheet gives a headline total at a fixed cell and daily NEW-follower + // rows after a fixed number of header rows. Both offsets are assumptions + // about a layout LinkedIn can change without telling anyone, and the + // headline total is assumed to be the count as of the LAST daily row - + // which the file never states. If any of that is wrong, every + // followers_at_post is off by the same amount and every reach_rate is + // biased in the same direction: the worst kind of error, invisible and + // systematic. + // + // So the assumption is checked before it is used. Walking back from the + // total must land on a non-negative baseline, because you cannot have + // gained more followers inside the window than you have in total. A + // negative baseline, a non-numeric headline, or a daily block that yields + // no rows at all all mean the layout or the window is not what this code + // thinks, and each is caught here. + // + // On failure the follower map is left empty, which nulls followers_at_post + // and reach_rate for every post. Falling back to the daily sheet alone is + // not an option: it carries GAINS, not LEVELS, so it cannot produce a + // reach_rate at all. A missing reach_rate is honest; a biased one is not. const f = sheets.FOLLOWERS ?? []; if (f.length) { - const total = Math.round(+f[0][1] || 0); + const total = Math.round(Number(f[0]?.[1])); const daily = []; for (const r of f.slice(3)) { const d = parseUsDate(r[0]); const n = Number(r[1]); if (d && Number.isFinite(n)) daily.push([d, n]); + else if (String(r[0] ?? '').trim()) dropped.follower.push(String(r[0])); } daily.sort((a, b) => a[0].localeCompare(b[0])); - let running = total; - for (let i = daily.length - 1; i >= 0; i--) { - followersByDate.set(daily[i][0], running); - running -= daily[i][1]; + const gained = daily.reduce((s, [, n]) => s + n, 0); + const baseline = total - gained; + const slack = Math.max(1, Math.abs(total) * FOLLOWER_TOTAL_TOLERANCE); + + if (!Number.isFinite(total) || total <= 0) { + warnings.push( + 'The FOLLOWERS sheet has no readable total where one is expected, so the follower ' + + 'count at post time cannot be reconstructed. followers_at_post and reach_rate are null.', + ); + } else if (!daily.length) { + warnings.push( + 'The FOLLOWERS sheet has a total but no readable daily rows, so the follower count at ' + + 'post time cannot be reconstructed. followers_at_post and reach_rate are null.', + ); + } else if (baseline < -slack) { + warnings.push( + `The FOLLOWERS sheet is internally inconsistent: it reports ${gained} new followers ` + + `over the window but a total of only ${total}, so the total is not the count as of the ` + + `last daily row. Reconstructing from it would bias every reach_rate by a constant, so ` + + 'followers_at_post and reach_rate are null instead.', + ); + } else { + let running = total; + for (let i = daily.length - 1; i >= 0; i--) { + followersByDate.set(daily[i][0], running); + running -= daily[i][1]; + } } } } + describeDrops(dropped, warnings); + const followerDates = [...followersByDate.keys()].sort(); const followersAt = (d) => { - if (!followerDates.length) return null; let best = null; for (const k of followerDates) { if (k <= d) best = k; else break; } - return followersByDate.get(best ?? followerDates[0]); + // A post older than every follower row has no measured count. Carrying the + // earliest reconstructed value backwards would be a guess wearing the + // costume of a measurement, and reach_rate would silently inherit it. + return best === null ? null : followersByDate.get(best); }; // collisions make the date join ambiguous; report rather than guess @@ -302,7 +402,8 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { if (collisions.length) { warnings.push( `${collisions.length} day(s) have more than one post. The exports cannot be joined on URL, ` + - `so metrics for those days cannot be attributed to a specific post and are dropped.`, + `so nothing that belongs to one specific post - metrics, post URL, media type, media count - ` + + `can be attributed on those days and all of it is left null.`, ); } @@ -311,14 +412,23 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { const day = p.date.slice(0, 10); const ambiguous = (perDay.get(day) ?? 0) > 1; const m = ambiguous ? {} : (metrics.get(day) ?? {}); - const kinds = media.get(day) ?? new Map(); - const mediaItems = [...kinds.values()].reduce((a, b) => a + b, 0); - const media_type = kinds.has('video') ? 'video' + // Rich_Media rows carry a date and nothing else that identifies a post, so + // on a colliding day an upload cannot be tied to one of the day's posts any + // more than an analytics row can. media_type is a covariate in every model + // downstream, so a guess here quietly contaminates the results; unknown is + // the only defensible value. + const kinds = ambiguous ? null : (media.get(day) ?? new Map()); + const mediaItems = kinds ? [...kinds.values()].reduce((a, b) => a + b, 0) : null; + const media_type = !kinds ? null + : kinds.has('video') ? 'video' : (kinds.has('feed document') || kinds.has('document')) ? 'document' : mediaItems > 1 ? 'carousel' : mediaItems === 1 ? 'single_image' : 'text_only'; + // Exempt from the collision guard on purpose: the follower count is a + // per-DAY level, not a per-post attribute, so two posts sharing a date + // genuinely share the same value. Nothing is being guessed by keeping it. const followers = followersAt(day); const impressions = m.impressions ?? null; const engagements = m.engagements ?? null; @@ -338,8 +448,13 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { impressions, engagements, followers_at_post: followers, - reach_rate: impressions && followers ? +(impressions / followers).toFixed(4) : null, - conversion_rate: impressions && engagements ? +(engagements / impressions).toFixed(5) : null, + // Zero engagements on a real post is the single most informative outcome + // in the file - it is the clearest evidence the writing did not land. A + // truthiness test here would null it out, and since the drop is not + // random it would pull every correlation toward "everything I write is + // fine". Only a zero denominator is genuinely undefined. + reach_rate: ratio(impressions, followers, 4), + conversion_rate: ratio(engagements, impressions, 5), has_metrics: impressions !== null || engagements !== null, days_since_previous_post: gap, weekday: new Date(p.date).toLocaleDateString('en-US', { weekday: 'long' }), @@ -354,5 +469,5 @@ export function buildCorpus(rawDir, { sinceYears = null } = {}) { ); } - return { items, warnings, sources: { sharesPath, richPath, xlsxPath } }; + return { items, warnings, dropped, sources: { sharesPath, richPath, xlsxPath } }; } diff --git a/src/cli.mjs b/src/cli.mjs index 4931f38..ff55e58 100755 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -7,6 +7,7 @@ * build parse exports into a corpus * rate send the corpus through the rubric pack * analyze correlate ratings against outcomes, honestly + * dashboard render report.json as one self-contained HTML page * run all of the above */ @@ -18,12 +19,15 @@ import { linkedinPostPack } from './rubrics/linkedin-post.mjs'; import { validatePack, toApiQuestions, buildState, countByTier, tiersOf } from './rubrics/pack.mjs'; import { powerReport, testDimension, benjaminiHochberg } from './stats.mjs'; import { buildReport } from './report.mjs'; +import { buildDashboard } from './dashboard.mjs'; import { directions as linkedinDirections } from './rubrics/linkedin-post.mjs'; const OUT_DIR = process.env.JEV_WRITER_OUT ?? 'jev-out'; const PACKS = { 'linkedin-post': linkedinPostPack }; // Direction of merit per dimension, required by the report layer. const DIRECTIONS = { 'linkedin-post': linkedinDirections }; +// Display name per pack, so the dashboard can name the platform in its copy. +const PLATFORMS = { 'linkedin-post': 'LinkedIn' }; const c = { dim: (s) => `\x1b[2m${s}\x1b[0m`, @@ -216,11 +220,23 @@ function analyze({ packId = 'linkedin-post' } = {}) { } const outcomes = ['conversion_rate', 'reach_rate']; - const nTests = Object.keys(dims).length * outcomes.length; + + // The family size announced to the reader must be the family size the + // correction actually used. A dimension without enough paired data was never + // tested, so counting it inflates the burden the correction appears to have + // carried. Mirrors the same calculation in src/report.mjs. + const testableFor = (outcome) => { + const subset = [...items.values()].filter((i) => i[outcome] != null && ratings.some((r) => r.id === i.id)); + return Object.entries(dims).filter(([, byId]) => { + const vals = subset.map((i) => byId[i.id]).filter((v) => v != null); + return vals.length >= 15 && new Set(vals).size >= 3; + }); + }; for (const outcome of outcomes) { const subset = [...items.values()].filter((i) => i[outcome] != null && ratings.some((r) => r.id === i.id)); - const power = powerReport(subset.length, nTests); + const testable = testableFor(outcome); + const power = powerReport(subset.length, testable.length); console.log(c.bold(`\n${'='.repeat(74)}\n${outcome}\n${'='.repeat(74)}`)); console.log((power.verdict === 'descriptive' ? c.red : power.verdict === 'weak' ? c.yellow : c.green)(power.headline)); @@ -230,17 +246,22 @@ function analyze({ packId = 'linkedin-post' } = {}) { continue; } - const control = subset.map((i) => new Date(i.date).getTime()); const rows = []; - for (const [name, byId] of Object.entries(dims)) { - const pairs = subset.map((i) => [byId[i.id], i[outcome]]).filter(([a]) => a != null); - if (pairs.length < 15) continue; - const vals = pairs.map(([a]) => a); + for (const [name, byId] of testable) { + // The publication date travels with its own item. Slicing a separate + // control array to the surviving length takes the first n dates, not the + // dates of the n surviving posts, so one missing rating misaligns every + // date after it. See the same fix in src/report.mjs. + const triples = subset + .map((i) => [byId[i.id], i[outcome], new Date(i.date).getTime()]) + .filter(([a]) => a != null); + if (triples.length < 15) continue; + const vals = triples.map((t) => t[0]); if (new Set(vals).size < 3) continue; const row = testDimension({ name, tier: tiers[name] ?? 'code', - values: vals, outcome: pairs.map(([, b]) => b), - control: control.slice(0, pairs.length), + values: vals, outcome: triples.map((t) => t[1]), + control: triples.map((t) => t[2]), }); if (row) rows.push(row); } @@ -299,6 +320,31 @@ function writeReport(packId) { } } +/** + * Render the dashboard. Reads report.json rather than recomputing, so what you + * see is exactly what the report layer concluded. The page embeds every post in + * full, which is why it is written to the output directory and never committed. + */ +function dashboard(packId = 'linkedin-post') { + const file = out('report.json'); + if (!existsSync(file)) { + throw new Error(`no ${file}. Run "jev-writer analyze" first.`); + } + const report = JSON.parse(readFileSync(file, 'utf8')); + const ratings = existsSync(out('ratings.jsonl')) ? readJsonl(out('ratings.jsonl')) : []; + const html = buildDashboard(report, { + ratings, + pack: PACKS[packId], + platform: PLATFORMS[packId] ?? null, + }); + const dest = out('dashboard.html'); + writeFileSync(dest, html); + const kb = Math.round(Buffer.byteLength(html) / 1024); + console.log(c.bold(`\n wrote ${dest}`) + ` (${kb} KB, ${report.posts.length} posts)`); + console.log(c.dim(' Self-contained: no network, no build step. Open it in any browser.')); + console.log(c.yellow(' It embeds the full text of every post. Do not commit it.')); +} + // ---------------------------------------------------------------- main const [cmd, arg] = process.argv.slice(2); @@ -307,7 +353,8 @@ try { else if (cmd === 'build') build(arg); else if (cmd === 'rate') await rate(); else if (cmd === 'analyze') analyze(); - else if (cmd === 'run') { build(arg); await rate(); analyze(); } + else if (cmd === 'dashboard') dashboard(); + else if (cmd === 'run') { build(arg); await rate(); analyze(); dashboard(); } else { console.log(`jev-writer @@ -315,7 +362,8 @@ try { build parse platform exports in into a corpus rate rate the corpus with the rubric pack analyze correlate ratings against outcomes, with a power gate - run build, rate, analyze + dashboard render report.json as one self-contained HTML page + run build, rate, analyze, dashboard Set exactly one of AI_GATEWAY_API_KEY, OPENROUTER_API_KEY, TYPESAFE_API_KEY. Output goes to ${OUT_DIR}/ (override with JEV_WRITER_OUT).`); diff --git a/src/dashboard.mjs b/src/dashboard.mjs new file mode 100644 index 0000000..f6d8765 --- /dev/null +++ b/src/dashboard.mjs @@ -0,0 +1,1004 @@ +/** + * Build the dashboard: one self-contained HTML file with inline CSS, inline JS + * and the data embedded. No CDN, no fetch, no build step. It opens over + * file:// on a laptop with no network. + * + * The presentation contract, which is the whole point of this file: + * - no statistics vocabulary anywhere except the collapsed methods section; + * - every rubric judgment appears as a coloured band, never as a raw number; + * - bands arrive direction-aware from the report and are never recomputed + * here, so a low score on a dimension where low is good still paints green; + * - charts only where a chart genuinely says something. + * + * This module does no I/O. It takes the object `buildReport` returned and gives + * back a string. The CLI owns reading and writing files. + */ + +import { DIMENSION_LABELS } from './report.mjs'; + +const labelFor = (d) => DIMENSION_LABELS[d] ?? d.replace(/_/g, ' '); + +/** + * Judgments are grouped into families for the expanded post view. A pack may + * declare its own `groups`; anything the pack does not place falls into a final + * catch-all so a new pack renders sensibly on day one instead of silently + * dropping half its questions. + */ +function groupsFor(pack, dims) { + const declared = Array.isArray(pack?.groups) ? pack.groups : []; + const placed = new Set(declared.flatMap((g) => g.dims)); + const leftover = dims.filter((d) => !placed.has(d)); + const groups = declared + .map((g) => ({ ...g, dims: g.dims.filter((d) => dims.includes(d)) })) + .filter((g) => g.dims.length); + if (leftover.length) groups.push({ key: 'other', title: groups.length ? 'Other judgments' : 'Judgments', dims: leftover }); + return groups; +} + +/** + * Text arriving from a CSV export often carries stray wrapping quotes and runs + * of blank lines. Rendering that raw makes every post look broken. + */ +function cleanText(raw) { + if (!raw) return ''; + const lines = String(raw) + .split('\n') + .map((l) => l.trim().replace(/^"+/, '').replace(/"+$/, '').trim()); + const out = []; + for (const l of lines) { + if (!l && !out.length) continue; + if (!l && !out[out.length - 1]) continue; + out.push(l); + } + while (out.length && !out[out.length - 1]) out.pop(); + return out.join('\n'); +} + +const titleCase = (s) => s.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase()); + +/** Choice answers are not bands; they get shown as plain descriptive chips. */ +function choiceAnswers(ratings) { + const out = {}; + for (const [k, v] of Object.entries(ratings ?? {})) { + if (v && v.type === 'choice' && v.value != null) out[k] = titleCase(String(v.value)); + } + return out; +} + +/** + * Question counts for the methods appendix. Derived from the pack so a pack with + * a different number of questions describes itself correctly; falls back to + * counting what actually reached the report when no pack is supplied. + */ +function questionCounts(pack, dims) { + const qs = pack?.questions ? Object.values(pack.questions) : null; + if (!qs) return { total: dims.length, primary: null, exploratory: null }; + return { + total: qs.length, + primary: qs.filter((q) => q.tier === 'primary').length, + exploratory: qs.filter((q) => q.tier === 'exploratory').length, + }; +} + +/** + * Reshape the report into the payload the page embeds. Nothing here touches the + * filesystem: `report` is exactly what buildReport returned, and `ratings` is + * the rows from ratings.jsonl, needed only because Choice answers are + * descriptive labels rather than bands and so are dropped from the report model. + */ +function buildData(report, { ratings = [], pack = null, platform = null } = {}) { + const choices = new Map(); + for (const row of ratings) { + choices.set(row.id, { ...(choices.get(row.id) ?? {}), ...choiceAnswers(row.ratings) }); + } + const validated = report.validated.map((v) => v.dimension); + + // buildReport spreads the whole corpus item into report.posts, so post text + // and raw metrics are already on `p`. The private version of this file joined + // a second posts.jsonl here; that join was redundant and is gone. + const slim = report.posts.map((p) => { + const text = cleanText(p.text || ''); + return { + id: p.id, + date: p.date, + url: p.post_url || p.share_link || null, + text, + snippet: text.replace(/\s+/g, ' ').slice(0, 190), + words: p.word_count ?? null, + media: p.media_type ?? null, + hasMetrics: !!p.has_metrics, + impressions: p.impressions ?? null, + engagements: p.engagements ?? null, + followers: p.followers_at_post ?? null, + reach: p.reach_rate ?? null, + conv: p.conversion_rate ?? null, + grade: p.grade ? p.grade.score : null, + bands: Object.fromEntries(Object.entries(p.bands).map(([k, b]) => [k, [b.band, b.label, b.hint]])), + choices: choices.get(p.id) ?? {}, + fixes: p.fixes ?? [], + }; + }); + + const allDims = [...new Set(report.posts.flatMap((p) => Object.keys(p.bands ?? {})))]; + const groups = groupsFor(pack, allDims); + const dimLabels = {}; + for (const d of allDims) dimLabels[d] = labelFor(d); + for (const g of groups) for (const d of g.dims) dimLabels[d] = labelFor(d); + + return { + generatedAt: report.generatedAt, + pack: report.pack, + summary: report.summary, + corpus: report.corpus, + outcomes: report.outcomes, + findings: report.findings, + weakSpots: report.weakSpots, + validated, + power: report.power, + writingPrompt: report.writingPrompt, + groups, + platform, + questionCounts: questionCounts(pack, allDims), + corpusAvailable: !!report.corpus, + dimLabels, + posts: slim, + }; +} + +/* ------------------------------------------------------------------ charts */ + +const esc = (s) => + String(s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); + +/** Posting volume by year. `muteIndex` dims the peak year when the story is a decline. */ +function volumeChart(rows, muteIndex = -1) { + const W = 640, H = 240, padL = 8, padR = 8, padT = 28, padB = 34; + const max = Math.max(...rows.map((r) => r.posts), 1); + const plotW = W - padL - padR, plotH = H - padT - padB; + const slot = plotW / rows.length; + const bw = Math.min(96, slot * 0.56); + let bars = ''; + rows.forEach((r, i) => { + const h = Math.max(2, (r.posts / max) * plotH); + const x = padL + slot * i + (slot - bw) / 2; + const y = padT + plotH - h; + const hi = i === muteIndex; + bars += + `` + + `${r.posts}` + + `${esc(r.year)}`; + }); + return ( + `` + + `` + + bars + + `` + ); +} + +/** Follower growth over time. */ +function followerChart(points) { + const W = 640, H = 220, padL = 46, padR = 12, padT = 18, padB = 30; + if (points.length < 2) return ''; + const xs = points.map((p) => new Date(p.date).getTime()); + const ys = points.map((p) => p.followers); + const x0 = Math.min(...xs), x1 = Math.max(...xs); + const y0 = Math.min(...ys), y1 = Math.max(...ys); + const plotW = W - padL - padR, plotH = H - padT - padB; + const px = (t) => padL + ((t - x0) / (x1 - x0 || 1)) * plotW; + const py = (v) => padT + plotH - ((v - y0) / (y1 - y0 || 1)) * plotH; + const d = points.map((p, i) => `${i ? 'L' : 'M'}${px(new Date(p.date).getTime()).toFixed(1)} ${py(p.followers).toFixed(1)}`).join(''); + const area = `${d}L${px(x1).toFixed(1)} ${(padT + plotH).toFixed(1)}L${px(x0).toFixed(1)} ${(padT + plotH).toFixed(1)}Z`; + const years = [...new Set(points.map((p) => String(p.date).slice(0, 4)))]; + let ticks = ''; + for (const y of years) { + const first = points.find((p) => String(p.date).slice(0, 4) === y); + ticks += `${esc(y)}`; + } + const fmt = (v) => (v >= 1000 ? `${(v / 1000).toFixed(1)}k` : String(v)); + return ( + `` + + `` + + `` + + `${fmt(y1)}` + + `${fmt(y0)}` + + `` + + ticks + + `` + ); +} + +/* -------------------------------------------------------------------- html */ + +function css() { + return ` +:root{ + --bg:#f7f7f8; --surface:#ffffff; --surface-2:#fbfbfc; --border:#e4e4e7; --border-strong:#d4d4d8; + --fg:#18181b; --fg-muted:#6b7280; --fg-faint:#9ca3af; + --accent:#4f46e5; --accent-fg:#ffffff; --accent-soft:rgba(79,70,229,.10); + --ok:#15803d; --ok-bg:rgba(22,163,74,.12); --ok-br:rgba(22,163,74,.30); + --good:#0f766e; --good-bg:rgba(13,148,136,.12); --good-br:rgba(13,148,136,.30); + --mid:#6b7280; --mid-bg:rgba(107,114,128,.12); --mid-br:rgba(107,114,128,.28); + --warn:#b45309; --warn-bg:rgba(217,119,6,.14); --warn-br:rgba(217,119,6,.32); + --bad:#b91c1c; --bad-bg:rgba(220,38,38,.12); --bad-br:rgba(220,38,38,.30); + --shadow:0 1px 2px rgba(16,18,27,.05),0 8px 24px -16px rgba(16,18,27,.24); + --radius:14px; +} +@media (prefers-color-scheme: dark){ + :root:not([data-theme="light"]){ + --bg:#09090b; --surface:#131316; --surface-2:#1a1a1f; --border:#26262b; --border-strong:#35353c; + --fg:#f4f4f5; --fg-muted:#a1a1aa; --fg-faint:#71717a; + --accent:#818cf8; --accent-fg:#0b0b10; --accent-soft:rgba(129,140,248,.14); + --ok:#4ade80; --ok-bg:rgba(74,222,128,.13); --ok-br:rgba(74,222,128,.28); + --good:#2dd4bf; --good-bg:rgba(45,212,191,.13); --good-br:rgba(45,212,191,.28); + --mid:#a1a1aa; --mid-bg:rgba(161,161,170,.13); --mid-br:rgba(161,161,170,.26); + --warn:#fbbf24; --warn-bg:rgba(251,191,36,.14); --warn-br:rgba(251,191,36,.30); + --bad:#f87171; --bad-bg:rgba(248,113,113,.14); --bad-br:rgba(248,113,113,.30); + --shadow:0 1px 2px rgba(0,0,0,.5),0 12px 32px -20px rgba(0,0,0,.9); + } +} +:root[data-theme="dark"]{ + --bg:#09090b; --surface:#131316; --surface-2:#1a1a1f; --border:#26262b; --border-strong:#35353c; + --fg:#f4f4f5; --fg-muted:#a1a1aa; --fg-faint:#71717a; + --accent:#818cf8; --accent-fg:#0b0b10; --accent-soft:rgba(129,140,248,.14); + --ok:#4ade80; --ok-bg:rgba(74,222,128,.13); --ok-br:rgba(74,222,128,.28); + --good:#2dd4bf; --good-bg:rgba(45,212,191,.13); --good-br:rgba(45,212,191,.28); + --mid:#a1a1aa; --mid-bg:rgba(161,161,170,.13); --mid-br:rgba(161,161,170,.26); + --warn:#fbbf24; --warn-bg:rgba(251,191,36,.14); --warn-br:rgba(251,191,36,.30); + --bad:#f87171; --bad-bg:rgba(248,113,113,.14); --bad-br:rgba(248,113,113,.30); + --shadow:0 1px 2px rgba(0,0,0,.5),0 12px 32px -20px rgba(0,0,0,.9); +} + +*,*::before,*::after{box-sizing:border-box} +html{-webkit-text-size-adjust:100%} +body{ + margin:0; background:var(--bg); color:var(--fg); + font-family:ui-sans-serif,-apple-system,BlinkMacSystemFont,"Segoe UI",Inter,Roboto,Helvetica,Arial,sans-serif; + font-size:16px; line-height:1.6; -webkit-font-smoothing:antialiased; overflow-x:hidden; +} +.num,.tnum{font-variant-numeric:tabular-nums; font-feature-settings:"tnum" 1} +.wrap{max-width:1080px; margin:0 auto; padding:0 16px} +a{color:var(--accent)} + +/* header */ +.top{position:sticky; top:0; z-index:20; background:color-mix(in srgb,var(--bg) 88%,transparent); + backdrop-filter:saturate(1.4) blur(10px); border-bottom:1px solid var(--border)} +.top-in{display:flex; align-items:center; gap:12px; min-height:58px} +.brand{font-weight:650; letter-spacing:-.01em; margin-right:auto; font-size:15px} +.brand small{display:block; font-weight:450; color:var(--fg-faint); font-size:12px; letter-spacing:0} +.tabs{display:flex; gap:2px; overflow-x:auto; scrollbar-width:none} +.tabs::-webkit-scrollbar{display:none} +.tabs a{white-space:nowrap; font-size:13px; color:var(--fg-muted); text-decoration:none; + padding:6px 10px; border-radius:8px} +.tabs a:hover{background:var(--surface-2); color:var(--fg)} +.iconbtn{flex:none; border:1px solid var(--border); background:var(--surface); color:var(--fg-muted); + width:34px; height:34px; border-radius:9px; cursor:pointer; font-size:15px; line-height:1} +.iconbtn:hover{color:var(--fg); border-color:var(--border-strong)} + +/* hero */ +.hero{padding:44px 0 8px} +h1{font-size:clamp(28px,6vw,40px); line-height:1.12; letter-spacing:-.025em; margin:0 0 10px; font-weight:680} +.lede{font-size:clamp(16px,2.4vw,18px); color:var(--fg-muted); max-width:62ch; margin:0} + +section{padding:36px 0 8px; scroll-margin-top:70px} +h2{font-size:clamp(20px,3.4vw,26px); letter-spacing:-.02em; margin:0 0 6px; font-weight:660} +h3{font-size:15px; letter-spacing:-.01em; margin:0 0 10px; font-weight:620} +.sub{color:var(--fg-muted); margin:0 0 20px; max-width:66ch; font-size:15px} +.eyebrow{font-size:11px; font-weight:650; letter-spacing:.09em; text-transform:uppercase; + color:var(--fg-faint); margin:0 0 8px} + +.card{background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); + box-shadow:var(--shadow)} +.pad{padding:20px} +.grid{display:grid; gap:14px} +.stats{grid-template-columns:repeat(auto-fit,minmax(min(210px,100%),1fr))} +.two{grid-template-columns:repeat(auto-fit,minmax(min(320px,100%),1fr))} + +.stat .k{font-size:12px; font-weight:600; color:var(--fg-muted); letter-spacing:.01em} +.stat .v{font-size:clamp(28px,5vw,34px); font-weight:680; letter-spacing:-.03em; line-height:1.15; margin:6px 0 6px} +.stat .d{font-size:13px; color:var(--fg-muted); line-height:1.5} +.stat .v .nodata{font-size:.5em;font-weight:500;opacity:.55;letter-spacing:0} +.unit{font-size:16px; font-weight:560; color:var(--fg-muted); margin-left:2px} + +/* charts */ +.chart{width:100%; height:auto; display:block; overflow:visible} +.bar{fill:var(--accent)} +.bar-muted{fill:var(--mid); opacity:.55} +.bar-val{fill:var(--fg); font-size:13px; font-weight:620; font-variant-numeric:tabular-nums} +.bar-lab{fill:var(--fg-faint); font-size:11.5px} +.axis{stroke:var(--border-strong)} +.gridline{stroke:var(--border); stroke-dasharray:3 4} +.line{fill:none; stroke:var(--accent); stroke-width:2.25; stroke-linejoin:round; stroke-linecap:round} +.area{fill:var(--accent-soft); stroke:none} +.note{font-size:13px; color:var(--fg-muted); margin:14px 0 0} + +/* findings */ +.finding{display:flex; gap:14px; align-items:flex-start; padding:20px} +.finding + .finding{border-top:1px solid var(--border)} +.finding .dot{flex:none; width:9px; height:9px; border-radius:50%; background:var(--ok); margin-top:11px; + box-shadow:0 0 0 4px var(--ok-bg)} +.finding p{margin:0; font-size:clamp(16px,2.3vw,19px); line-height:1.5; letter-spacing:-.012em} +.finding .meta{margin-top:8px; font-size:13px; color:var(--fg-muted)} +.soft .dot{background:var(--mid); box-shadow:0 0 0 4px var(--mid-bg)} +.soft p{font-size:15px; color:var(--fg-muted)} + +.gap{padding:18px 20px} +.gap + .gap{border-top:1px solid var(--border)} +.gap p{margin:0 0 12px; font-size:15.5px; line-height:1.5} +.gaprow{display:flex; align-items:center; gap:10px; margin-bottom:6px} +.gaprow .gl{flex:none; width:118px; font-size:12px; color:var(--fg-muted)} +.gaptrack{flex:1; min-width:0; height:9px; border-radius:99px; background:var(--surface-2); + border:1px solid var(--border); overflow:hidden} +.gaptrack i{display:block; height:100%; border-radius:99px} +.gaptrack i.typ{background:var(--mid); opacity:.55} +.gaptrack i.win{background:var(--ok)} +.gapnote{font-size:12px; color:var(--fg-faint); margin-top:8px} +@media (max-width:560px){ .gaprow{flex-wrap:wrap; gap:4px} .gaprow .gl{width:auto} .gaptrack{flex:1 1 100%} } + +/* chips */ +.chip{display:inline-flex; align-items:center; gap:6px; border-radius:99px; padding:3px 10px; + font-size:12px; font-weight:560; border:1px solid var(--border); background:var(--surface-2); + color:var(--fg-muted); white-space:nowrap; max-width:100%} +.chip .cl{overflow:hidden; text-overflow:ellipsis} +.chip-excellent{color:var(--ok); background:var(--ok-bg); border-color:var(--ok-br)} +.chip-strong{color:var(--good); background:var(--good-bg); border-color:var(--good-br)} +.chip-typical{color:var(--mid); background:var(--mid-bg); border-color:var(--mid-br)} +.chip-weak{color:var(--warn); background:var(--warn-bg); border-color:var(--warn-br)} +.chip-poor{color:var(--bad); background:var(--bad-bg); border-color:var(--bad-br)} +.chips{display:flex; flex-wrap:wrap; gap:6px} + +/* controls */ +.controls{display:flex; flex-wrap:wrap; gap:8px; align-items:center; margin-bottom:16px} +select,input[type=search],button.btn{ + font:inherit; font-size:13.5px; color:var(--fg); background:var(--surface); + border:1px solid var(--border); border-radius:9px; padding:7px 11px} +input[type=search]{flex:1 1 180px; min-width:0} +select:focus-visible,input:focus-visible,button:focus-visible,summary:focus-visible,.post:focus-visible{ + outline:2px solid var(--accent); outline-offset:2px} +button.btn{cursor:pointer; font-weight:560} +button.btn:hover{border-color:var(--border-strong)} +.count{font-size:13px; color:var(--fg-faint); margin-left:auto} + +/* posts */ +.posts{display:grid; gap:12px} +.post{background:var(--surface); border:1px solid var(--border); border-radius:var(--radius); + box-shadow:var(--shadow); overflow:hidden} +.post-head{display:flex; gap:16px; align-items:flex-start; padding:16px; cursor:pointer; width:100%; + background:none; border:0; text-align:left; color:inherit; font:inherit} +.post-head:hover{background:var(--surface-2)} +.gradebox{flex:none; width:62px; text-align:center} +.gradebox .g{font-size:27px; font-weight:700; letter-spacing:-.04em; line-height:1.05; + font-variant-numeric:tabular-nums} +.gradebox .gl{font-size:10.5px; font-weight:640; letter-spacing:.06em; text-transform:uppercase; margin-top:2px} +.g-strong,.gl-strong{color:var(--ok)} .g-good,.gl-good{color:var(--good)} +.g-mixed,.gl-mixed{color:var(--warn)} .g-weak,.gl-weak{color:var(--bad)} +.post-main{min-width:0; flex:1} +.post-meta{display:flex; flex-wrap:wrap; gap:10px; font-size:12px; color:var(--fg-faint); margin-bottom:5px} +.post-snip{margin:0 0 10px; font-size:14.5px; line-height:1.5; color:var(--fg); + display:-webkit-box; -webkit-line-clamp:2; -webkit-box-orient:vertical; overflow:hidden} +.metricline{font-size:12.5px; color:var(--fg-muted); margin-bottom:8px} +.metricline b{color:var(--fg); font-weight:620; font-variant-numeric:tabular-nums} +.caret{flex:none; color:var(--fg-faint); font-size:12px; margin-top:4px; transition:transform .15s ease} +.post.open .caret{transform:rotate(90deg)} +.detail{display:none; border-top:1px solid var(--border); padding:18px 16px; background:var(--surface-2)} +.post.open .detail{display:block} +.fulltext{white-space:pre-wrap; overflow-wrap:anywhere; font-size:14.5px; line-height:1.62; + margin:0 0 20px; color:var(--fg)} +.fixes{border:1px solid var(--accent-soft); background:var(--accent-soft); border-radius:11px; + padding:14px 16px; margin:0 0 20px} +.fixes ol{margin:8px 0 0; padding-left:20px} +.fixes li{font-size:14.5px; margin-bottom:3px} +.jgroup{margin-bottom:16px} +.jgroup h4{font-size:11px; font-weight:650; letter-spacing:.08em; text-transform:uppercase; + color:var(--fg-faint); margin:0 0 8px} +.jrow{display:flex; gap:10px; align-items:baseline; justify-content:space-between; + padding:6px 0; border-bottom:1px dashed var(--border)} +.jrow:last-child{border-bottom:0} +.jname{font-size:14px; min-width:0; overflow-wrap:anywhere} +.jval{flex:none; text-align:right} +.jhint{display:block; font-size:11.5px; color:var(--fg-faint); margin-top:2px} +.empty{padding:28px 16px; text-align:center; color:var(--fg-muted); font-size:14px} + +/* prompt */ +.promptbox{position:relative} +.promptbox pre{margin:0; padding:18px; white-space:pre-wrap; overflow-wrap:anywhere; + font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace; font-size:12.5px; line-height:1.62; + color:var(--fg); max-height:420px; overflow-y:auto} +.copybar{display:flex; gap:8px; align-items:center; padding:12px 16px; border-bottom:1px solid var(--border)} +.copybar .ok{font-size:13px; color:var(--ok)} + +/* methods */ +details.methods{margin-top:8px} +details.methods > summary{cursor:pointer; list-style:none; padding:16px 20px; font-weight:600; + display:flex; align-items:center; gap:10px; font-size:15px} +details.methods > summary::-webkit-details-marker{display:none} +details.methods > summary::before{content:"›"; color:var(--fg-faint); transition:transform .15s ease} +details.methods[open] > summary::before{transform:rotate(90deg)} +.methods-body{padding:0 20px 20px; font-size:14px; color:var(--fg-muted)} +.methods-body h4{font-size:13px; color:var(--fg); margin:18px 0 6px} +.methods-body code{font-family:ui-monospace,Menlo,monospace; font-size:12.5px; + background:var(--surface-2); border:1px solid var(--border); border-radius:5px; padding:1px 5px} +.methods-body ul{padding-left:20px; margin:6px 0} +footer{padding:40px 0 56px; color:var(--fg-faint); font-size:12.5px} + +@media (max-width:480px){ .brand small{display:none} } +@media (max-width:560px){ + .post-head{padding:14px; gap:12px} + .gradebox{width:52px} + .gradebox .g{font-size:23px} + .pad{padding:16px} + .count{margin-left:0; width:100%} +} +@media (prefers-reduced-motion: reduce){*{transition:none !important; animation:none !important}} +`; +} + +function appJs() { + return ` +(function(){ + var D = JSON.parse(document.getElementById("payload").textContent); + var root = document.documentElement; + + /* theme ------------------------------------------------------------- */ + function setTheme(t){ + root.setAttribute("data-theme", t); + try{ localStorage.setItem("lca-theme", t); }catch(e){} + var b = document.getElementById("themeBtn"); + if(b){ b.textContent = t === "dark" ? "☀" : "☾"; b.setAttribute("aria-label", "Switch to " + (t === "dark" ? "light" : "dark") + " mode"); } + } + var stored = null; + try{ stored = localStorage.getItem("lca-theme"); }catch(e){} + setTheme(stored || (window.matchMedia && window.matchMedia("(prefers-color-scheme: light)").matches ? "light" : "dark")); + document.getElementById("themeBtn").addEventListener("click", function(){ + setTheme(root.getAttribute("data-theme") === "dark" ? "light" : "dark"); + }); + + /* helpers ----------------------------------------------------------- */ + var MONTHS = ["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]; + function fmtDate(s){ var d = new Date(s); return d.getDate() + " " + MONTHS[d.getMonth()] + " " + d.getFullYear(); } + function pct(x, dp){ return (x*100).toFixed(dp === undefined ? 1 : dp) + "%"; } + function gradeTier(g){ return g >= 80 ? "strong" : g >= 60 ? "good" : g >= 40 ? "mixed" : "weak"; } + function gradeWord(g){ return g >= 80 ? "Strong" : g >= 60 ? "Good" : g >= 40 ? "Mixed" : "Weak"; } + function el(tag, cls, txt){ var e = document.createElement(tag); if(cls) e.className = cls; if(txt != null) e.textContent = txt; return e; } + function chip(bandKey, label, title){ + var c = el("span", "chip chip-" + bandKey); + var s = el("span", "cl", label); c.appendChild(s); + if(title) c.title = title; + return c; + } + + /* posts ------------------------------------------------------------- */ + var list = document.getElementById("postList"); + var countEl = document.getElementById("postCount"); + var sortSel = document.getElementById("sortSel"); + var filtSel = document.getElementById("filtSel"); + var q = document.getElementById("searchBox"); + var KEY = D.validated.slice(0, 5); + + function comparator(mode){ + switch(mode){ + case "grade-asc": return function(a,b){ return (a.grade==null?999:a.grade) - (b.grade==null?999:b.grade); }; + case "new": return function(a,b){ return b.date.localeCompare(a.date); }; + case "old": return function(a,b){ return a.date.localeCompare(b.date); }; + case "reach": return function(a,b){ return (b.reach||-1) - (a.reach||-1); }; + case "conv": return function(a,b){ return (b.conv||-1) - (a.conv||-1); }; + default: return function(a,b){ return (b.grade==null?-1:b.grade) - (a.grade==null?-1:a.grade); }; + } + } + + function buildDetail(p){ + var d = el("div", "detail"); + + if(p.fixes && p.fixes.length){ + var fx = el("div", "fixes"); + fx.appendChild(el("h3", null, "What would most improve this post")); + var ol = document.createElement("ol"); + p.fixes.forEach(function(dim){ + var b = p.bands[dim]; + var li = el("li", null, (D.dimLabels[dim] || dim) + (b ? " — currently " + b[1].toLowerCase() + " for you" : "")); + ol.appendChild(li); + }); + fx.appendChild(ol); + d.appendChild(fx); + } + + d.appendChild(el("h3", null, "Full post")); + d.appendChild(el("div", "fulltext", p.text || "(no text)")); + + d.appendChild(el("h3", null, "Every judgment on this post")); + D.groups.forEach(function(g){ + var rows = []; + g.dims.forEach(function(dim){ + var b = p.bands[dim]; + if(b){ + rows.push({ dim: dim, band: b[0], label: b[1], hint: b[2] }); + } else if(p.choices[dim]){ + rows.push({ dim: dim, choice: p.choices[dim] }); + } + }); + if(!rows.length) return; + var box = el("div", "jgroup"); + box.appendChild(el("h4", null, g.title)); + rows.forEach(function(r){ + var row = el("div", "jrow"); + var name = el("div", "jname", D.dimLabels[r.dim] || r.dim); + var val = el("div", "jval"); + if(r.choice){ + val.appendChild(chip("typical", r.choice)); + } else { + val.appendChild(chip(r.band, r.label)); + val.appendChild(el("span", "jhint", r.hint)); + } + row.appendChild(name); row.appendChild(val); + box.appendChild(row); + }); + d.appendChild(box); + }); + + if(p.url){ + var a = document.createElement("a"); + a.href = p.url; a.target = "_blank"; a.rel = "noreferrer noopener"; + a.textContent = "Open this post \u2197"; + a.style.fontSize = "13.5px"; + d.appendChild(a); + } + return d; + } + + function card(p){ + var wrap = el("article", "post"); + var head = document.createElement("button"); + head.className = "post-head"; head.type = "button"; + head.setAttribute("aria-expanded", "false"); + + var gb = el("div", "gradebox"); + if(p.grade == null){ + gb.appendChild(el("div", "g num", "–")); + gb.appendChild(el("div", "gl", "no grade")); + } else { + gb.appendChild(el("div", "g num g-" + gradeTier(p.grade), String(p.grade))); + gb.appendChild(el("div", "gl gl-" + gradeTier(p.grade), gradeWord(p.grade))); + } + head.appendChild(gb); + + var main = el("div", "post-main"); + var meta = el("div", "post-meta"); + meta.appendChild(el("span", "tnum", fmtDate(p.date))); + if(p.words != null) meta.appendChild(el("span", "tnum", p.words + " words")); + if(!p.hasMetrics) meta.appendChild(el("span", null, "no numbers recorded")); + main.appendChild(meta); + main.appendChild(el("p", "post-snip", p.snippet + (p.text.length > 190 ? "…" : ""))); + + if(p.hasMetrics && p.impressions != null){ + var m = el("div", "metricline"); + var bits = ["Seen by " + p.impressions.toLocaleString() + " people"]; + if(p.reach != null) bits.push("reach " + pct(p.reach) + " of your followers"); + if(p.conv == null) bits.push("reactions not recorded"); + else if(p.conv > 0 && p.conv < 0.001) bits.push("under 0.1% of viewers reacted"); + else bits.push("" + pct(p.conv) + " of viewers reacted"); + m.innerHTML = bits.join(" · "); + main.appendChild(m); + } + + var chips = el("div", "chips"); + KEY.forEach(function(dim){ + var b = p.bands[dim]; + if(!b) return; + chips.appendChild(chip(b[0], b[1] + " · " + (D.dimLabels[dim] || dim), b[2])); + }); + main.appendChild(chips); + head.appendChild(main); + head.appendChild(el("div", "caret", "›")); + wrap.appendChild(head); + + var built = false; + head.addEventListener("click", function(){ + if(!built){ wrap.appendChild(buildDetail(p)); built = true; } + var open = wrap.classList.toggle("open"); + head.setAttribute("aria-expanded", open ? "true" : "false"); + }); + return wrap; + } + + function render(){ + var term = (q.value || "").trim().toLowerCase(); + var rows = D.posts.filter(function(p){ + if(filtSel.value === "metrics" && !p.hasMetrics) return false; + if(filtSel.value === "top" && !(p.grade >= 80)) return false; + if(filtSel.value === "low" && !(p.grade != null && p.grade < 40)) return false; + if(term && p.text.toLowerCase().indexOf(term) === -1) return false; + return true; + }); + rows.sort(comparator(sortSel.value)); + list.textContent = ""; + if(!rows.length){ list.appendChild(el("div", "empty card", "No posts match that.")); } + else { + var frag = document.createDocumentFragment(); + rows.forEach(function(p){ frag.appendChild(card(p)); }); + list.appendChild(frag); + } + countEl.textContent = rows.length + " of " + D.posts.length + " posts"; + } + + sortSel.addEventListener("change", render); + filtSel.addEventListener("change", render); + q.addEventListener("input", render); + render(); + + /* copy prompt -------------------------------------------------------- */ + var copyBtn = document.getElementById("copyBtn"); + if(copyBtn){ + copyBtn.addEventListener("click", function(){ + var text = document.getElementById("promptText").textContent; + var done = function(){ + var s = document.getElementById("copyState"); + s.textContent = "Copied"; + setTimeout(function(){ s.textContent = ""; }, 1800); + }; + if(navigator.clipboard && navigator.clipboard.writeText){ + navigator.clipboard.writeText(text).then(done, fallback); + } else { fallback(); } + function fallback(){ + var ta = document.createElement("textarea"); + ta.value = text; ta.style.position = "fixed"; ta.style.opacity = "0"; + document.body.appendChild(ta); ta.select(); + try{ document.execCommand("copy"); done(); }catch(e){} + document.body.removeChild(ta); + } + }); + } +})(); +`; +} + +/** + * Turn the derived cadence into a heading and a sentence. Every branch has to be + * true of the data that produced it; where there is not enough history to say + * anything honest, it says only what is countable. + */ +function volumeStory(cadence, years) { + const total = years.reduce((a, y) => a + y.posts, 0); + if (!cadence) { + return { title: 'How often you published', body: `${total} posts across ${years.length || 1} year${years.length === 1 ? '' : 's'}.`, mute: -1 }; + } + const { direction, peakYear, peakPosts, latestYear, latestPosts, otherYearAverage } = cadence; + const muted = years.findIndex((y) => y.year === peakYear); + const tail = 'Everything else on this page is about quality. How often you show up is the other half.'; + if (direction === 'collapsed') { + return { + title: 'Your posting volume fell away', + body: `${peakPosts} posts in ${peakYear}, then around ${otherYearAverage} a year since. ${tail}`, + mute: muted, + }; + } + if (direction === 'declined') { + return { + title: 'You are publishing less than you used to', + body: `${peakPosts} posts in ${peakYear}, ${latestPosts} in ${latestYear}. ${tail}`, + mute: muted, + }; + } + if (direction === 'rising') { + return { + title: 'You are publishing more than you used to', + body: `${latestPosts} posts in ${latestYear}, your busiest year so far. ${tail}`, + mute: -1, + }; + } + return { + title: 'Your posting volume is steady', + body: `Around ${otherYearAverage} posts a year, ${latestPosts} in ${latestYear}. ${tail}`, + mute: -1, + }; +} + +function html(data) { + const c = data.corpus; + const confirmed = data.findings.filter((f) => f.confirmed); + const soft = data.findings.filter((f) => !f.confirmed); + const years = c.volumeByYear ?? []; + const followerGain = c.followerEnd != null && c.followerStart != null ? c.followerEnd - c.followerStart : null; + // The volume story is derived, never assumed. A hard-coded "you stopped + // posting" heading was true of the one corpus this was first built against + // and is a false accusation against anyone whose output grew. + const volume = volumeStory(c.cadence, years); + const counts = data.questionCounts; + const platform = data.platform; + const plat = platform ? esc(platform) : 'the platform'; + // null * 100 is 0 in JS, so an unguarded formatter turns "we have no + // engagement data" into the assertion "your posts got 0.0% reach". That is a + // fabricated finding, which is the one thing this project must not produce. + const pct1 = (x) => (x == null || !Number.isFinite(x) ? null : `${(x * 100).toFixed(1)}`); + const dateRange = data.summary.dateRange.map((d) => (d ? String(d).slice(0, 10) : '?')); + + const findingCard = (f, soften) => ` +
+ +
+

${esc(f.sentence)}

+
About ${esc(f.outcomeShort === 'reach' ? 'how far it was distributed' : 'what people did once they saw it')}
+
+
`; + + const gapCard = (w) => { + // A relative bar, never the rubric number itself. Both bars share a scale, + // so the visible difference IS the gap the sentence describes. + const span = Math.max(Math.abs(w.typical), Math.abs(w.winners), 0.001); + const bar = (v) => Math.max(3, Math.min(100, (Math.abs(v) / span) * 100)).toFixed(1); + return ` +
+

${esc(w.advice)}

+
+ Your typical post +
+
+
+ Your best posts +
+
+
Longer bar means more ${esc(w.label)}.
+
`; + }; + + const payload = JSON.stringify(data).replace(/[<\u2028\u2029]/g, (c) => '\\u' + c.charCodeAt(0).toString(16).padStart(4, '0')); + + return ` + + + + +Content Review + + + + + + +
+
+
Content Review${esc(dateRange[0])} – ${esc(dateRange[1])}
+ + +
+
+ +
+ +
+

What your writing actually did

+

${data.summary.posts} posts, read and judged one at a time, then checked against the ones where + ${plat} gave you real numbers. No jargon below — just what worked, what did not, and what to do next.

+
+ + +
+

Section one

+

How your content is doing

+

Two things shape every post: how far the platform pushed it, and what people did once they saw it. + They are worth reading separately.

+ +
+
+
Posts published
+
${data.summary.posts}
+
Since ${esc(dateRange[0])}. ${data.summary.withOutcomes} of them have view and reaction + numbers attached — the rest predate the export.
+
+
+
Typical reach
+
${pct1(c.medianReachRate) ?? 'No data'}${pct1(c.medianReachRate) == null ? '' : '%'}
+
${esc(data.outcomes.reach_rate.explainer)}
+
+
+
Typical engagement
+
${pct1(c.medianConversionRate) ?? 'No data'}${pct1(c.medianConversionRate) == null ? '' : '%'}
+
${esc(data.outcomes.conversion_rate.explainer)}
+
+
+
Followers
+
${(c.followerEnd ?? 0).toLocaleString('en-US')}
+
${followerGain != null ? `${followerGain > 0 ? 'Up' : 'Down'} ${Math.abs(followerGain).toLocaleString('en-US')}` : 'Change unknown'} since your first post here.
+
+
+ +
+
+

${esc(volume.title)}

+

${esc(volume.body)}

+ ${volumeChart(years, volume.mute)} +

Posts published per year.

+
+
+

${followerGain != null && followerGain > 0 ? 'Your audience kept growing' : 'Your audience over time'}

+

From ${(c.followerStart ?? 0).toLocaleString('en-US')} to + ${(c.followerEnd ?? 0).toLocaleString('en-US')} over the same period.

+ ${followerChart(c.followers)} +

Followers at the time of each post.

+
+
+
+ + +
+

Section two

+

What's working for you

+

These come from comparing your own posts against each other — the ones that did well against + the ones that did not. They are about your audience, not about ${plat} in general.

+ +
+ ${confirmed.length ? confirmed.map((f) => findingCard(f, false)).join('') : '
Nothing held up firmly enough to state as a result.
'} +
+ + ${soft.length ? ` +

Worth watching

+

Patterns that showed up, but in fewer posts and without the same weight behind them. Treat these + as hunches to test, not as rules.

+
+ ${soft.map((f) => findingCard(f, true)).join('')} +
` : ''} + +

The gap between your best posts and your typical post

+

Same person, same account — but your winners consistently do more of some things and less of + others. Closing these gaps is the most concrete thing on this page.

+
+ ${data.weakSpots.length ? data.weakSpots.map(gapCard).join('') : '
Not enough posts with numbers to compare.
'} +
+
+ + +
+

Section three

+

Your posts, graded

+

Every post gets a mark out of 100, built only from the things that actually moved your numbers. + Colours are relative to your own writing: green means better than most of your posts, amber and red mean worse. + Tap any post to see the full text and every judgment made about it.

+ +
+ + + + + +
+ +
+
+ + +
+

Section four

+

Your writing prompt

+

This was written from your own results — the things that earned engagement from your audience, + and the habits your best posts avoid. Paste it into any assistant along with a draft, before you publish. + It is not generic advice; every check in it came out of your own posts.

+
+
+ + + scroll inside the box for the rest +
+ 
${esc(data.writingPrompt ?? 'No prompt could be generated.')}
+
+
+ + +
+
+
+ How this was worked out +
+

Everything above avoids technical language on purpose. This section does not — it is here so the + numbers can be checked rather than trusted.

+ +

What was measured

+

Each of the ${data.summary.posts} posts was rated against the ${esc(data.pack.id)} rubric pack + (v${esc(data.pack.version)}), ${counts.total} questions in all, each answered independently several + times and averaged. ${counts.primary} ${counts.primary === 1 ? 'question was' : 'questions were'} + pre-registered as primary hypotheses; the remaining ${counts.exploratory} are exploratory and are never + promoted to a result.

+ +

Bands

+

A raw rubric level is meaningless on its own, so every dimension is banded against the quintiles of this + author's own distribution: bottom 20% is Poor, top 20% is Excellent. Each question + carries a declared direction of merit, so ai_generated_feel bands in reverse to + hook_strength, and dimensions marked neutral (technicality, voice confidence, + vulnerability, provokes disagreement) always band as Typical rather than being scored as good + or bad.

+ +

Statistical detail

+

Each dimension was tested against two outcomes — reach rate (impressions / followers at post time) and + conversion rate (engagements / impressions) — using a partial Spearman rank correlation controlling for + post date, with p-values from a permutation test on the date-residualised ranks, and a 0.05 threshold. + Only dimensions clearing that threshold enter the post grade, weighted by |rho|. Findings from + pre-registered primary questions are shown as results; exploratory ones appear as "worth watching".

+
    +
  • Reach: ${esc(data.power.reach_rate.headline)}
    • +
  • Engagement: ${esc(data.power.conversion_rate.headline)}
    • +
+

Findings, with the numbers behind them

+
    + ${data.findings + .map( + (f) => + `
  • ${esc(f.dimension)} vs ${esc(f.outcome)} — ` + + `${f.confirmed ? 'primary' : 'exploratory'}, partial rho ${f._stats.partialRho.toFixed(3)}, ` + + `p = ${f._stats.p.toFixed(4)}, n = ${f._stats.n}; top third median ` + + `${(f.comparison.highMedian * 100).toFixed(1)}% vs bottom third ` + + `${(f.comparison.lowMedian * 100).toFixed(1)}%.
    • `, + ) + .join('')} +
+ +

Caveats worth taking seriously

+
    +
  • Only ${data.summary.withOutcomes} of ${data.summary.posts} posts have outcome data, and they are not + a random sample — they are the recent ones. Anything before ${esc(String(years[1]?.year ?? ''))} is + graded but never contributed to a finding.
    • +
  • With this many posts and this many tests, roughly + ${data.power.reach_rate.expectedFalsePositives.toFixed(1)} of + ${data.power.reach_rate.nTests} tests will clear the threshold by chance alone. That is why exploratory + results are separated out rather than mixed in.
    • +
  • Rubric ratings come from a language model reading the text. They are consistent, not objective.
    • +
  • Everything here is a correlation within one person's posting history. It cannot prove that changing + a dimension would change the outcome.
    • +
+
+
+
+
+ +
+ Generated ${esc(String(data.generatedAt).slice(0, 10))} from your own export. Private — everything on this page + stays in this file. +
+
+ + + + + +`; +} + +/** + * Build the dashboard as a single HTML string. + * + * @param {object} report exactly what buildReport() returned + * @param {object} [opts] + * @param {Array} [opts.ratings] rows from ratings.jsonl, for Choice answers + * @param {object} [opts.pack] the rubric pack, for question groups and counts + * @param {string} [opts.platform] display name, e.g. "LinkedIn"; omitted stays generic + * @returns {string} a self-contained HTML document + */ +export function buildDashboard(report, opts = {}) { + if (!report || !Array.isArray(report.posts)) { + throw new Error('buildDashboard needs the object buildReport() returned'); + } + if (!report.corpus) { + // A report.json written before corpusFacts existed. Say which file is stale + // rather than letting the first property access throw somewhere opaque. + throw new Error( + 'this report.json predates the dashboard and has no "corpus" field. Re-run "jev-writer analyze".', + ); + } + return html(buildData(report, opts)); +} diff --git a/src/index.mjs b/src/index.mjs new file mode 100644 index 0000000..e9d2533 --- /dev/null +++ b/src/index.mjs @@ -0,0 +1,50 @@ +// Library entry point. The CLI is the other consumer of these same modules; +// nothing here is CLI-specific, so an agent or a notebook can drive the whole +// pipeline without shelling out. +// +// The subpath exports in package.json are the stable surface. Anything not +// re-exported here or listed there is internal and may change without a major. + +export { + rank, + pearson, + spearman, + residualise, + partialSpearman, + permutationP, + bootstrapCI, + benjaminiHochberg, + criticalR, + minimumDetectableEffect, + powerReport, + testDimension, +} from './stats.mjs'; + +export { + BANDS, + bandEdges, + bandFor, + gradePost, + buildReport, + buildWritingPrompt, + DIMENSION_LABELS, + labelFor, +} from './report.mjs'; + +export { + TIERS, + validatePack, + toApiQuestions, + tiersOf, + countByTier, + buildState, + score, + choice, + bool, +} from './rubrics/pack.mjs'; + +export { buildCorpus, textFeatures, splitHook } from './adapters/linkedin.mjs'; + +export { PROVIDERS, detectProvider, availableProviders, evaluate } from './providers/index.mjs'; + +export { buildDashboard } from './dashboard.mjs'; diff --git a/src/report.mjs b/src/report.mjs index 4a13fed..adc2011 100644 --- a/src/report.mjs +++ b/src/report.mjs @@ -203,6 +203,15 @@ const OUTCOMES = { * Returns plain data. Nothing here renders, so any front end - the bundled * one, a Next.js app, a terminal table - consumes the same structure. */ +/** + * Below MIN_PAIRS paired observations a rank correlation is not worth running, + * and below MIN_DISTINCT levels the dimension has not discriminated anything. + * Both gates decide what counts as a test, so they are named once and shared + * with the family-size calculation rather than repeated as literals. + */ +const MIN_PAIRS = 15; +const MIN_DISTINCT = 3; + export function buildReport({ items, ratings, pack, directions }) { const byId = new Map(ratings.map((r) => [r.id, r.ratings])); const rated = items.filter((i) => byId.has(i.id)); @@ -224,24 +233,52 @@ export function buildReport({ items, ratings, pack, directions }) { // Which dimensions actually predicted something, per outcome. const validatedByOutcome = {}; const findings = []; - const nTests = dims.length * Object.keys(OUTCOMES).length; + + /** + * A dimension is only a TEST against an outcome if there is enough paired + * data to run one. Announcing `dims.length * outcomes` while correcting over + * a smaller set overstates the burden the correction carried: a dimension + * dropped for thin coverage was never tested, and a reader told that 48 tests + * were corrected should not be looking at a correction over 6. + * + * So the testable set is computed first, per outcome, and the SAME number is + * both announced by the power report and used as the family size for + * Benjamini-Hochberg below. + */ + const testableFor = (outcome) => { + const subset = rated.filter((i) => i[outcome] != null); + return dims.filter((d) => { + const vals = subset.map((i) => byId.get(i.id)?.[d]?.value).filter((v) => v != null); + return vals.length >= MIN_PAIRS && new Set(vals).size >= MIN_DISTINCT; + }); + }; for (const [outcome, meta] of Object.entries(OUTCOMES)) { const subset = rated.filter((i) => i[outcome] != null); - const power = powerReport(subset.length, nTests); + const testable = testableFor(outcome); + const power = powerReport(subset.length, testable.length); validatedByOutcome[outcome] = []; if (power.verdict === 'descriptive') continue; - const control = subset.map((i) => new Date(i.date).getTime()); const candidates = []; - for (const d of dims) { - const pairs = subset.map((i) => [byId.get(i.id)?.[d]?.value, i[outcome]]).filter(([a]) => a != null); - if (pairs.length < 15) continue; - const a = pairs.map(([x]) => x), b = pairs.map(([, y]) => y); - if (new Set(a).size < 3) continue; - const partial = partialSpearman(a, b, control.slice(0, pairs.length)); - const resA = residualise(rank(a), rank(control.slice(0, pairs.length))); - const resB = residualise(rank(b), rank(control.slice(0, pairs.length))); + for (const d of testable) { + // Carry the control value alongside its own item. An earlier version + // built \`control\` over the whole subset and then took + // \`control.slice(0, pairs.length)\`, which is the FIRST n dates rather + // than the dates OF the surviving items. Any item dropped for a missing + // rating shifted every later date onto the wrong post, silently + // corrupting the date control that decides whether a finding is + // confirmed. Keeping the triple together makes the misalignment + // unrepresentable. + const triples = subset + .map((i) => [byId.get(i.id)?.[d]?.value, i[outcome], new Date(i.date).getTime()]) + .filter(([a]) => a != null); + if (triples.length < MIN_PAIRS) continue; + const a = triples.map((t) => t[0]), b = triples.map((t) => t[1]), control = triples.map((t) => t[2]); + if (new Set(a).size < MIN_DISTINCT) continue; + const partial = partialSpearman(a, b, control); + const resA = residualise(rank(a), rank(control)); + const resB = residualise(rank(b), rank(control)); const p = permutationP(resA, resB); const tier = pack.questions[d].tier; if (Number.isNaN(partial)) continue; @@ -372,18 +409,92 @@ export function buildReport({ items, ratings, pack, directions }) { dateRange: [items[0]?.date ?? null, items[items.length - 1]?.date ?? null], }, power: Object.fromEntries( - Object.keys(OUTCOMES).map((o) => [o, powerReport(rated.filter((i) => i[o] != null).length, nTests)]), + Object.keys(OUTCOMES).map((o) => [o, powerReport(rated.filter((i) => i[o] != null).length, testableFor(o).length)]), ), outcomes: OUTCOMES, findings: findings.sort((a, b) => Number(b.confirmed) - Number(a.confirmed) || b.comparison.ratio - a.comparison.ratio), validated, weakSpots, bandEdges: edgesByDim, + corpus: corpusFacts(items), posts, writingPrompt: buildWritingPrompt({ findings, weakSpots, pack }), }; } +/** + * Posting volume, follower growth, and headline medians, taken straight off the + * corpus rather than off the ratings. This is the part of a dashboard that needs + * no model and no statistics: it is just what the author did. + * + * `cadence` is derived rather than asserted. An earlier version of the dashboard + * hard-coded the narrative "you stopped posting", which was true of exactly one + * corpus and reads as a false accusation against anyone whose volume grew. + */ +export function corpusFacts(items) { + const byYear = new Map(); + for (const i of items) { + const y = String(i.date).slice(0, 4); + const row = byYear.get(y) ?? { year: y, posts: 0, withMetrics: 0 }; + row.posts++; + if (i.has_metrics) row.withMetrics++; + byYear.set(y, row); + } + const volumeByYear = [...byYear.values()].sort((a, b) => a.year.localeCompare(b.year)); + + const followers = items + .filter((i) => i.followers_at_post != null) + .map((i) => ({ date: i.date, followers: i.followers_at_post })); + + const withMetrics = items.filter((i) => i.has_metrics); + const med = (xs) => median(xs.filter((v) => v != null)); + + return { + volumeByYear, + cadence: cadenceOf(volumeByYear), + followers, + followerStart: followers[0]?.followers ?? null, + followerEnd: followers[followers.length - 1]?.followers ?? null, + medianReachRate: med(withMetrics.map((i) => i.reach_rate)), + medianConversionRate: med(withMetrics.map((i) => i.conversion_rate)), + medianImpressions: med(withMetrics.map((i) => i.impressions)), + totalImpressions: withMetrics.reduce((a, i) => a + (i.impressions ?? 0), 0), + totalEngagements: withMetrics.reduce((a, i) => a + (i.engagements ?? 0), 0), + }; +} + +/** + * Describe how posting volume moved, so the dashboard can write a true sentence + * about it instead of a templated one. Returns null when there is not enough + * history to say anything honest (a single year is a data point, not a trend). + */ +export function cadenceOf(volumeByYear) { + if (volumeByYear.length < 2) return null; + // `>=` so a tie resolves to the LATER year. With a strict `>` the first of two + // equal years won, the later one fell into `rest`, dragged the average down, + // and an author whose latest year matched their best year was told they were + // publishing less. That is the exact false accusation this function exists to + // avoid, reintroduced through a comparison operator. + const peak = volumeByYear.reduce((a, b) => (b.posts >= a.posts ? b : a)); + const rest = volumeByYear.filter((r) => r.year !== peak.year); + const recent = volumeByYear[volumeByYear.length - 1]; + const others = rest.length ? rest.reduce((a, r) => a + r.posts, 0) / rest.length : peak.posts; + // A third of peak is the line between "eased off" and "essentially stopped". + const ratio = peak.posts === 0 ? 1 : others / peak.posts; + const direction = + recent.posts >= peak.posts + ? (rest.length && peak.posts > others ? "rising" : "steady") + : ratio < 0.34 ? "collapsed" : ratio < 0.75 ? "declined" : "steady"; + return { + direction, + peakYear: peak.year, + peakPosts: peak.posts, + latestYear: recent.year, + latestPosts: recent.posts, + otherYearAverage: Math.round(others * 10) / 10, + }; +} + /** * A prompt the author can paste into any assistant before publishing the next * draft. It is generated from THEIR results: the dimensions that predicted diff --git a/src/rubrics/linkedin-post.mjs b/src/rubrics/linkedin-post.mjs index fe519d9..552082f 100644 --- a/src/rubrics/linkedin-post.mjs +++ b/src/rubrics/linkedin-post.mjs @@ -34,6 +34,16 @@ export const linkedinPostPack = { reach_rate: 'impressions / followers at post time - did the platform distribute it?', conversion_rate: 'engagements / impressions - given it was seen, did it work?', }, + // Families for the expanded post view in the dashboard. Presentation only: + // grouping never touches how a question is asked, rated or correlated. A pack + // that declares none still renders, with every question in one list. + groups: [ + { key: 'attention', title: 'Attention', dims: ['preview_earns_expansion', 'hook_strength', 'payoff_delivery', 'hook_archetype'] }, + { key: 'craft', title: 'Craft', dims: ['reading_ease', 'skimmability', 'technicality', 'specificity', 'voice_confidence', 'cliche_density', 'ai_generated_feel', 'post_format'] }, + { key: 'substance', title: 'Substance', dims: ['actionability', 'novelty', 'shelf_life', 'has_firsthand_evidence', 'teaches_transferable_skill', 'ending_type'] }, + { key: 'social', title: 'How it behaves socially', dims: ['provokes_disagreement', 'who_is_the_hero', 'vulnerability', 'emotional_register', 'is_self_promotional', 'is_engagement_bait', 'is_humblebrag'] }, + { key: 'verdict', title: 'Overall verdicts', dims: ['is_slop', 'provides_real_value', 'human_authorship_evidence', 'algorithmic_penalty_risk', 'most_needed_improvement'] }, + ], questions: { preview_earns_expansion: score( diff --git a/src/stats.mjs b/src/stats.mjs index b6180c7..5a3bcb6 100644 --- a/src/stats.mjs +++ b/src/stats.mjs @@ -92,8 +92,27 @@ function mulberry32(seed) { }; } -/** Permutation test. Exact enough at these n, and assumes nothing about shape. */ -export function permutationP(a, b, iterations = 10000, seed = 20260920) { +/** + * Seed for every randomised procedure in this module. + * + * A permutation p-value and a bootstrap CI both wobble from run to run. Fixing + * the seed makes a published finding reproducible by anyone holding the same + * corpus, which is the point of pre-registering the rubric in the first place. + * It is overridable so a result can be re-run under a different stream: a + * finding that only survives on one seed was never a finding. + */ +export const DEFAULT_SEED = 20260920; + +/** + * Permutation test. Exact enough at these n, and assumes nothing about shape. + * + * The fourth argument is `{ seed }`. A bare number is still accepted because + * this function is exported from the package root and the seed used to be + * positional; dropping that would change callers' numbers instead of breaking + * them, which is the one failure mode this module exists to prevent. + */ +export function permutationP(a, b, iterations = 10000, options = {}) { + const { seed = DEFAULT_SEED } = typeof options === 'number' ? { seed: options } : options; const observed = Math.abs(spearman(a, b)); if (Number.isNaN(observed)) return NaN; const rnd = mulberry32(seed); @@ -111,7 +130,7 @@ export function permutationP(a, b, iterations = 10000, seed = 20260920) { } /** Bootstrap percentile CI. Normal approximations are not trustworthy at n<60. */ -export function bootstrapCI(a, b, { iterations = 5000, alpha = 0.05, seed = 20260920 } = {}) { +export function bootstrapCI(a, b, { iterations = 5000, alpha = 0.05, seed = DEFAULT_SEED } = {}) { const rnd = mulberry32(seed); const n = a.length; const out = []; @@ -202,19 +221,23 @@ export function powerReport(n, nTests) { * Run one dimension against one outcome and return an honest row. * `tier` carries the pre-registration: 'primary' results are evidence, * 'exploratory' results are hypotheses for the next batch of data. + * + * The seed travels out on the row rather than staying implicit, so a reader of + * the report can reproduce the p-value and the interval without having to know + * what this module's default happened to be on the day it was run. */ -export function testDimension({ name, tier, values, outcome, control = null }) { +export function testDimension({ name, tier, values, outcome, control = null, seed = DEFAULT_SEED }) { const n = values.length; const rho = spearman(values, outcome); if (Number.isNaN(rho)) return null; - const [lo, hi] = bootstrapCI(values, outcome); - const p = permutationP(values, outcome); - const row = { name, tier, n, rho, ci: [lo, hi], p }; + const [lo, hi] = bootstrapCI(values, outcome, { seed }); + const p = permutationP(values, outcome, 10000, { seed }); + const row = { name, tier, n, rho, ci: [lo, hi], p, seed }; if (control) { row.partialRho = partialSpearman(values, outcome, control); const rv = residualise(rank(values), rank(control)); const ro = residualise(rank(outcome), rank(control)); - row.partialP = permutationP(rv, ro); + row.partialP = permutationP(rv, ro, 10000, { seed }); row.survivesControl = row.partialP < 0.05; } return row; diff --git a/test/adapter.test.mjs b/test/adapter.test.mjs index c8b622f..e0dc3c8 100644 --- a/test/adapter.test.mjs +++ b/test/adapter.test.mjs @@ -50,3 +50,303 @@ test('countable features are counted in code', () => { assert.equal(f.emoji_count, 1); assert.ok(f.blank_line_count >= 1); }); + +// -------------------------------------------------------------------------- +// buildCorpus fixtures +// +// buildCorpus reads a directory, so these tests write real exports to a temp +// dir. The xlsx is assembled byte by byte rather than mocked: the join logic +// under test depends on the exact sheet layout (a headline total at one fixed +// cell, daily rows after three header rows), and a mock would let that layout +// drift without failing anything. +// -------------------------------------------------------------------------- + +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { buildCorpus } from '../src/adapters/linkedin.mjs'; + +/** Minimal STORED-method zip writer. Enough for readZipEntries, no crc needed. */ +function zip(files) { + const locals = [], centrals = []; + let offset = 0; + for (const [name, content] of Object.entries(files)) { + const nameBuf = Buffer.from(name, 'utf8'); + const data = Buffer.from(content, 'utf8'); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(data.length, 22); + local.writeUInt16LE(nameBuf.length, 26); + locals.push(local, nameBuf, data); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt32LE(data.length, 20); + central.writeUInt32LE(data.length, 24); + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt32LE(offset, 42); + centrals.push(central, nameBuf); + offset += 30 + nameBuf.length + data.length; + } + const body = Buffer.concat(locals); + const dir = Buffer.concat(centrals); + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(Object.keys(files).length, 8); + eocd.writeUInt16LE(Object.keys(files).length, 10); + eocd.writeUInt32LE(dir.length, 12); + eocd.writeUInt32LE(body.length, 16); + return Buffer.concat([body, dir, eocd]); +} + +const cell = (v) => `${v}`; +const sheetXml = (rows) => + '' + + rows.map((r) => `${r.map(cell).join('')}`).join('') + + ''; + +/** rows for TOP POSTS: [url, M/D/YYYY, engagements, _, url, M/D/YYYY, impressions, _]. */ +function xlsx({ topPosts = [], followers = [] } = {}) { + const sheets = { 'TOP POSTS': 'sheet1.xml', FOLLOWERS: 'sheet2.xml' }; + return zip({ + 'xl/workbook.xml': + '' + + '' + + '' + + '', + 'xl/_rels/workbook.xml.rels': + '' + + '' + + '' + + '', + [`xl/worksheets/${sheets['TOP POSTS']}`]: sheetXml([[''], [''], [''], ...topPosts]), + [`xl/worksheets/${sheets.FOLLOWERS}`]: sheetXml(followers), + }); +} + +/** [total, daily...] in the layout the adapter assumes: total at row 0 col 1, rows from 3. */ +const followerSheet = (total, daily) => + [['Total followers', total], ['', ''], ['Date', 'New followers'], ...daily]; + +const SHARES_HEADER = 'Date,ShareLink,ShareCommentary,Visibility\n'; +const shareRow = (date, text) => `${date},https://www.linkedin.com/feed/update/urn:li:ugcPost:1,"${text}",MEMBER_NETWORK\n`; + +const dirs = []; +function fixture({ shares = [], rich = null, book = null }) { + const dir = mkdtempSync(join(tmpdir(), 'jev-writer-')); + dirs.push(dir); + writeFileSync(join(dir, 'Shares.csv'), SHARES_HEADER + shares.map(([d, t]) => shareRow(d, t)).join('')); + if (rich) writeFileSync(join(dir, 'Rich_Media.csv'), rich); + if (book) writeFileSync(join(dir, 'Analytics_export.xlsx'), book); + return dir; +} + +process.on('exit', () => { for (const d of dirs) rmSync(d, { recursive: true, force: true }); }); + +const RICH_HEADER = 'Date/Time,Media Link\n'; +const richRow = (raw) => `"${raw}",https://media.example.com/1\n`; + +// -------------------------------------------------------------------------- +// same-date collisions +// -------------------------------------------------------------------------- + +test('a same-date collision leaves media unattributed, not guessed', () => { + const dir = fixture({ + shares: [ + ['2026-01-02 09:00:00', 'First post of the day about latency work.'], + ['2026-01-02 17:00:00', 'Second post of the day about hiring.'], + ['2026-01-05 09:00:00', 'A post on a day of its own.'], + ], + rich: RICH_HEADER + richRow('Member uploaded an image on January 2, 2026') + + richRow('Member uploaded an image on January 5, 2026'), + }); + const { items } = buildCorpus(dir); + const colliding = items.filter((i) => i.date.startsWith('2026-01-02')); + assert.equal(colliding.length, 2); + for (const i of colliding) { + assert.equal(i.media_type, null, 'media type cannot be attributed on a colliding day'); + assert.equal(i.media_item_count, null); + } + const alone = items.find((i) => i.date.startsWith('2026-01-05')); + assert.equal(alone.media_type, 'single_image'); + assert.equal(alone.media_item_count, 1); +}); + +test('a same-date collision leaves metrics and post_url null', () => { + const dir = fixture({ + shares: [ + ['2026-01-02 09:00:00', 'First post of the day about latency work.'], + ['2026-01-02 17:00:00', 'Second post of the day about hiring.'], + ], + book: xlsx({ + topPosts: [['https://li/p/a', '1/2/2026', '40', '', 'https://li/p/a', '1/2/2026', '900', '']], + followers: followerSheet(1000, [['1/1/2026', 5], ['1/2/2026', 5]]), + }), + }); + const { items } = buildCorpus(dir); + for (const i of items) { + assert.equal(i.impressions, null); + assert.equal(i.engagements, null); + assert.equal(i.post_url, null); + assert.equal(i.has_metrics, false); + } +}); + +test('followers_at_post survives a collision because it is a per-day level', () => { + const dir = fixture({ + shares: [ + ['2026-01-02 09:00:00', 'First post of the day about latency work.'], + ['2026-01-02 17:00:00', 'Second post of the day about hiring.'], + ], + book: xlsx({ followers: followerSheet(1000, [['1/1/2026', 5], ['1/2/2026', 5]]) }), + }); + const { items } = buildCorpus(dir); + assert.equal(items[0].followers_at_post, 1000); + assert.equal(items[1].followers_at_post, 1000); +}); + +// -------------------------------------------------------------------------- +// unreadable dates +// -------------------------------------------------------------------------- + +test('post rows with an unreadable date are counted and quoted, not silently dropped', () => { + const dir = fixture({ + shares: [ + ['2026-01-02 09:00:00', 'A post that parses.'], + ['not a date', 'A post whose date is junk.'], + ['02/03/2026', 'A post in the wrong date format.'], + ], + }); + const { items, warnings, dropped } = buildCorpus(dir); + assert.equal(items.length, 1); + assert.deepEqual(dropped.post, ['not a date', '02/03/2026']); + const w = warnings.find((x) => /post row\(s\)/.test(x)); + assert.match(w, /^2 post row\(s\) in Shares_\*\.csv had an unreadable date/); + assert.match(w, /"not a date", "02\/03\/2026"/); +}); + +test('unreadable media and analytics dates are surfaced as warnings too', () => { + const dir = fixture({ + shares: [['2026-01-02 09:00:00', 'A post that parses.']], + rich: RICH_HEADER + richRow('Member uploaded an image sometime last spring'), + book: xlsx({ topPosts: [['https://li/p/a', 'Jan 2 2026', '40', '', '', '', '', '']] }), + }); + const { warnings, dropped } = buildCorpus(dir); + assert.deepEqual(dropped.media, ['Member uploaded an image sometime last spring']); + assert.deepEqual(dropped.metric, ['Jan 2 2026']); + assert.ok(warnings.some((w) => /1 row\(s\) in Rich_Media\.csv had an unreadable date/.test(w))); + assert.ok(warnings.some((w) => /1 analytics row\(s\) under TOP POSTS had an unreadable date/.test(w))); +}); + +test('buildCorpus warns about bad dates instead of throwing', () => { + const dir = fixture({ shares: [['nonsense', 'Every single row is broken.']] }); + const { items, warnings } = buildCorpus(dir); + assert.equal(items.length, 0); + assert.ok(warnings.some((w) => /unreadable date/.test(w))); +}); + +// -------------------------------------------------------------------------- +// the follower back-walk +// -------------------------------------------------------------------------- + +test('a follower total contradicted by the daily rows nulls the derived fields', () => { + // 500 gained inside a window that ends at a total of 100 is impossible, which + // proves the headline is not the count as of the last daily row. + const dir = fixture({ + shares: [['2026-01-02 09:00:00', 'A post with impressions.']], + book: xlsx({ + topPosts: [['https://li/p/a', '1/2/2026', '40', '', 'https://li/p/a', '1/2/2026', '900', '']], + followers: followerSheet(100, [['1/1/2026', 200], ['1/2/2026', 300]]), + }), + }); + const { items, warnings } = buildCorpus(dir); + assert.equal(items[0].followers_at_post, null); + assert.equal(items[0].reach_rate, null); + assert.equal(items[0].impressions, 900, 'impressions themselves are still trustworthy'); + assert.ok(warnings.some((w) => /internally inconsistent/.test(w))); +}); + +test('a follower sheet whose layout has moved is rejected rather than trusted', () => { + const dir = fixture({ + shares: [['2026-01-02 09:00:00', 'A post with impressions.']], + book: xlsx({ + topPosts: [['https://li/p/a', '1/2/2026', '40', '', 'https://li/p/a', '1/2/2026', '900', '']], + // Headline cell now holds a label, as it would after a layout change. + followers: [['Followers', 'Total'], ['', ''], ['Date', 'New followers'], ['1/1/2026', 5]], + }), + }); + const { items, warnings } = buildCorpus(dir); + assert.equal(items[0].followers_at_post, null); + assert.equal(items[0].reach_rate, null); + assert.ok(warnings.some((w) => /no readable total/.test(w))); +}); + +test('a consistent follower sheet is walked back and used', () => { + const dir = fixture({ + shares: [['2026-03-02 09:00:00', 'A post with impressions.']], + book: xlsx({ + topPosts: [['https://li/p/a', '3/2/2026', '40', '', 'https://li/p/a', '3/2/2026', '500', '']], + followers: followerSheet(1000, [['3/1/2026', 10], ['3/2/2026', 20]]), + }), + }); + const { items } = buildCorpus(dir); + assert.equal(items[0].followers_at_post, 1000); + assert.equal(items[0].reach_rate, 0.5); +}); + +test('posts older than the follower window get no follower count at all', () => { + // The earliest reconstructed count is a measurement of March, not of January. + const dir = fixture({ + shares: [ + ['2026-01-02 09:00:00', 'A post from before the follower window.'], + ['2026-03-02 09:00:00', 'A post from inside it.'], + ], + book: xlsx({ + topPosts: [ + ['https://li/p/a', '1/2/2026', '40', '', 'https://li/p/a', '1/2/2026', '500', ''], + ['https://li/p/b', '3/2/2026', '40', '', 'https://li/p/b', '3/2/2026', '500', ''], + ], + followers: followerSheet(1000, [['3/1/2026', 10], ['3/2/2026', 20]]), + }), + }); + const { items } = buildCorpus(dir); + assert.equal(items[0].followers_at_post, null); + assert.equal(items[0].reach_rate, null); + assert.equal(items[1].followers_at_post, 1000); + assert.equal(items[1].reach_rate, 0.5); +}); + +// -------------------------------------------------------------------------- +// zero is an outcome +// -------------------------------------------------------------------------- + +test('zero engagements is an outcome, not a missing value', () => { + // The clearest evidence a post failed must not be the one row excluded from + // every correlation. + const dir = fixture({ + shares: [['2026-03-02 09:00:00', 'A post nobody reacted to.']], + book: xlsx({ + topPosts: [['https://li/p/a', '3/2/2026', '0', '', 'https://li/p/a', '3/2/2026', '500', '']], + followers: followerSheet(1000, [['3/1/2026', 10], ['3/2/2026', 20]]), + }), + }); + const { items } = buildCorpus(dir); + assert.equal(items[0].engagements, 0); + assert.equal(items[0].conversion_rate, 0); + assert.equal(items[0].has_metrics, true); +}); + +test('zero impressions leaves the ratios undefined rather than infinite', () => { + const dir = fixture({ + shares: [['2026-03-02 09:00:00', 'A post nobody saw.']], + book: xlsx({ + topPosts: [['https://li/p/a', '3/2/2026', '0', '', 'https://li/p/a', '3/2/2026', '0', '']], + followers: followerSheet(1000, [['3/1/2026', 10], ['3/2/2026', 20]]), + }), + }); + const { items } = buildCorpus(dir); + assert.equal(items[0].impressions, 0); + assert.equal(items[0].conversion_rate, null); + assert.equal(items[0].reach_rate, 0); +}); diff --git a/test/dashboard.test.mjs b/test/dashboard.test.mjs new file mode 100644 index 0000000..b8fc2c2 --- /dev/null +++ b/test/dashboard.test.mjs @@ -0,0 +1,131 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildDashboard } from '../src/dashboard.mjs'; +import { buildReport, HIGHER_IS_BETTER, LOWER_IS_BETTER } from '../src/report.mjs'; +import { score, bool } from '../src/rubrics/pack.mjs'; + +function corpus(n, { volume = 'flat' } = {}) { + const pack = { + id: 'test', version: '1.0.0', stateFields: ['text'], + groups: [{ key: 'craft', title: 'Craft', dims: ['hook_strength'] }], + questions: { + hook_strength: score('primary', { question: 'How strong is the hook in `text`?' }, + [{ summary: 'weak', signals: ['nothing'] }, { summary: 'strong', signals: ['something'] }]), + is_slop: bool('exploratory', 'Does `text` read as slop?'), + }, + }; + const directions = { hook_strength: HIGHER_IS_BETTER, is_slop: LOWER_IS_BETTER }; + const items = [], ratings = []; + for (let i = 0; i < n; i++) { + // `volume` controls which year each post lands in, so the derived narrative + // can be exercised in both directions. The splits are deliberately uneven: + // three equal years classify as "steady" and would test nothing. + const f = i / n; + const year = volume === 'growth' ? (f < 0.1 ? 2023 : f < 0.3 ? 2024 : 2025) + : volume === 'collapse' ? (f < 0.8 ? 2023 : f < 0.9 ? 2024 : 2025) + : 2025; + items.push({ + id: `p${i}`, date: `${year}-0${(i % 9) + 1}-1${i % 9}`, + text: `post number ${i} `, + has_metrics: true, impressions: 1000, engagements: 10 + i, followers_at_post: 500 + i, + reach_rate: 2, conversion_rate: (10 + i) / 1000, + }); + ratings.push({ id: `p${i}`, ratings: { + hook_strength: { type: 'score', value: i % 7 }, + is_slop: { type: 'boolean', value: ((i * 5) % 13) / 13 }, + } }); + } + return { items, ratings, pack, directions }; +} + +const render = (n, opts) => { + const { items, ratings, pack, directions } = corpus(n, opts); + return buildDashboard(buildReport({ items, ratings, pack, directions }), { ratings, pack, platform: 'TestNet' }); +}; + +test('produces one self-contained document with no network dependencies', () => { + const html = render(30); + assert.match(html, /^/i); + assert.match(html, /<\/html>\s*$/i); + // The whole point: it opens over file:// on a laptop with no network. + assert.doesNotMatch(html, /]+\ssrc=/i, 'an external script was linked'); + assert.doesNotMatch(html, /]+stylesheet/i, 'an external stylesheet was linked'); + assert.doesNotMatch(html, /https?:\/\/(cdn|unpkg|fonts|ajax)\./i, 'a CDN host appears in the output'); +}); + +test('post text is escaped, not injected', () => { + const html = render(30); + assert.doesNotMatch(html, /