e1_oracle.cpp

// e1_oracle.cpp -- Decoder-mirrored ORACLE for exploit E1 (entity completion).
//
// Single pass over enwik9 in file order. Maintains an online population (trie)
// of EXACT RAW byte strings previously inserted from:
//   (a) <title>...</title> contents
//   (b) completed internal-link targets: bytes between "[[" and first "|" or "]]"
//       (raw; no normalization/entity decoding; skip if len==0 or >256 or has '\n')
//
// For each link-target occurrence, BEFORE inserting it, replay its bytes L->R.
// At byte position i (i>=1) with already-replayed prefix P = target[0..i-1]:
//   let node = trie node for P.  If it does not exist -> C==0 -> ABSTAIN.
//   else W = node.w (total insertion-count weight of prior strings with prefix P),
//        w = weight of prior strings with prefix P + target[i] (child weight).
//        escape e = 1/(W+1); distribution: cont symbol s gets (1-e)*w_s/W, escape gets e.
//        cost(hit,  w>0) = -log2((1-e)*w/W) = log2((W+1)/w)
//        cost(escape,w==0)= -log2(e)         = log2(W+1)   [actual next byte not a prior continuation]
//   DETERMINISTIC iff exactly one distinct continuation symbol (children + END) among prior strings.
// Position 0 (empty prefix) is a FORCED abstain, recorded separately.
// END event: after last byte, predict "string ends here" (a prior string == full target
//   contributes an END continuation) with the same escape model.
//
// Variant overlay (sensitivity, report sec.4): in the SAME pass, descend 3 normalized
// prefixes in parallel (first-char case-toggle; underscore<->space swap; both). When exact
// ABSTAINS, if a variant prefix still exists and predicts the (transformed) next byte, count
// a VARIANT-HIT priced at its log-loss + 1 bit (variant-flag placeholder).
//
// Build: export PATH="/d/Safe/Tools/w64devkit/bin:$PATH"
//        g++ -O3 -std=c++17 e1_oracle.cpp -o e1_oracle.exe -lpsapi
// Run:   ./e1_oracle.exe enwik9

#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <cstdint>
#include <cstdarg>
#include <cmath>
#include <vector>
#include <chrono>
#include <windows.h>
#include <psapi.h>
using namespace std;

static FILE* g_rep=nullptr;
static void L(const char* fmt, ...){ va_list ap; va_start(ap,fmt); vfprintf(g_rep,fmt,ap); va_end(ap); }

static const uint64_t EMPTY = ~0ull;

// ---- global edge hash: key=(parent<<8)|byte -> child node id -------------------
struct Hash {
    uint64_t* keys=nullptr; uint32_t* vals=nullptr;
    size_t cap=0, cnt=0, mask=0;
    void init(size_t c){ cap=c; mask=c-1; keys=(uint64_t*)malloc(c*8); vals=(uint32_t*)malloc(c*4);
        for(size_t i=0;i<c;i++) keys[i]=EMPTY; cnt=0; }
    static inline uint64_t mix(uint64_t x){ x^=x>>30; x*=0xbf58476d1ce4e5b9ull; x^=x>>27;
        x*=0x94d049bb133111ebull; x^=x>>31; return x; }
    inline int64_t get(uint64_t key) const {
        size_t i=mix(key)&mask;
        while(keys[i]!=EMPTY){ if(keys[i]==key) return (int64_t)vals[i]; i=(i+1)&mask; }
        return -1;
    }
    void grow(){
        size_t nc=cap*2, nmask=nc-1;
        uint64_t* nk=(uint64_t*)malloc(nc*8); uint32_t* nv=(uint32_t*)malloc(nc*4);
        for(size_t i=0;i<nc;i++) nk[i]=EMPTY;
        for(size_t i=0;i<cap;i++) if(keys[i]!=EMPTY){
            size_t j=mix(keys[i])&nmask; while(nk[j]!=EMPTY) j=(j+1)&nmask; nk[j]=keys[i]; nv[j]=vals[i]; }
        free(keys); free(vals); keys=nk; vals=nv; cap=nc; mask=nmask;
    }
    inline void put(uint64_t key, uint32_t val){
        if((cnt+1)*10 >= cap*6) grow();
        size_t i=mix(key)&mask;
        while(keys[i]!=EMPTY){ if(keys[i]==key){ vals[i]=val; return; } i=(i+1)&mask; }
        keys[i]=key; vals[i]=val; cnt++;
    }
};

// ---- trie node storage (SoA) ---------------------------------------------------
static vector<uint32_t> NW;   // total insertion-count weight of strings with this prefix
static vector<uint32_t> NE;   // weight of strings ending exactly here (END continuation)
static vector<uint16_t> NC;   // number of distinct child bytes
static Hash H;
static uint64_t g_distinct=0, g_inserts=0;

static inline uint32_t newnode(){ NW.push_back(0); NE.push_back(0); NC.push_back(0);
    return (uint32_t)(NW.size()-1); }
static inline int64_t child(uint32_t node, uint8_t b){ return H.get(((uint64_t)node<<8)|b); }

static void insert(const uint8_t* s, size_t len){
    uint32_t node=0; // root
    for(size_t k=0;k<len;k++){
        uint8_t b=s[k];
        uint64_t key=((uint64_t)node<<8)|b;
        int64_t c=H.get(key);
        uint32_t cc;
        if(c<0){ cc=newnode(); H.put(key, cc); NC[node]++; }
        else cc=(uint32_t)c;
        node=cc; NW[node]++;
    }
    if(NE[node]==0) g_distinct++;
    NE[node]++;
    g_inserts++;
}

static inline uint8_t togglecase(uint8_t b){
    if(b>='A'&&b<='Z') return b+32;
    if(b>='a'&&b<='z') return b-32;
    return b;
}
static inline uint8_t swapus(uint8_t b){
    if(b=='_') return ' ';
    if(b==' ') return '_';
    return b;
}

// ---- accumulators --------------------------------------------------------------
struct Stats {
    uint64_t occ=0, bytes=0;
    uint64_t pos0=0;                              // forced abstain at position 0
    uint64_t ab=0;                                // non-pos0 abstain bytes (exact C==0)
    uint64_t det_c=0, det_w=0, amb_h=0, esc_w=0;  // non-abstain byte outcomes
    double   cost=0.0;                            // bits over non-abstain byte events
    // END events
    uint64_t e_ab=0, e_det_c=0, e_det_w=0, e_amb_h=0, e_esc_w=0;
    double   e_cost=0.0;
    // variant overlay (subset of the ab bytes)
    uint64_t var_hit=0, var_still=0;
    double   var_cost=0.0;                        // bits incl +1 per hit
};
static void merge(Stats& a, const Stats& b){
    a.occ+=b.occ; a.bytes+=b.bytes; a.pos0+=b.pos0; a.ab+=b.ab;
    a.det_c+=b.det_c; a.det_w+=b.det_w; a.amb_h+=b.amb_h; a.esc_w+=b.esc_w; a.cost+=b.cost;
    a.e_ab+=b.e_ab; a.e_det_c+=b.e_det_c; a.e_det_w+=b.e_det_w; a.e_amb_h+=b.e_amb_h; a.e_esc_w+=b.e_esc_w; a.e_cost+=b.e_cost;
    a.var_hit+=b.var_hit; a.var_still+=b.var_still; a.var_cost+=b.var_cost;
}

static Stats G;                 // global
static Stats REGCON[4][3];      // region x construct (0=plain,1=category,2=interwiki)
static const char* CONNAME[3]={"plain link target","Category","interwiki"};
static const char* REGNAME[4]={"0-100 MB","100-400 MB","400-600 MB","600-1000 MB"};

static bool is_interwiki(const uint8_t* s, size_t len){
    size_t i=0, run=0;
    while(i<len && s[i]>='a'&&s[i]<='z'){ i++; run++; }
    if(run<2||run>3) return false;
    while(i<len && s[i]=='-'){ i++; size_t g=0; while(i<len && s[i]>='a'&&s[i]<='z'){ i++; g++; } if(g<1) return false; }
    return (i<len && s[i]==':');
}

// replay one target occurrence, accumulate, then insert
static void process(const uint8_t* s, size_t len, size_t occOff){
    Stats loc; loc.occ=1; loc.bytes=len;
    int64_t ne=0, n1=0, n2=0, n3=0; // node ids; -1 == dead branch; 0 == root
    for(size_t i=0;i<len;i++){
        uint8_t b=s[i];
        uint8_t b1=swapus(b);
        uint8_t b2=(i==0)?togglecase(b):b;
        uint8_t b3=(i==0)?togglecase(swapus(b)):swapus(b);
        int64_t ce=(ne>=0)?child((uint32_t)ne,b ):-1;
        int64_t c1=(n1>=0)?child((uint32_t)n1,b1):-1;
        int64_t c2=(n2>=0)?child((uint32_t)n2,b2):-1;
        int64_t c3=(n3>=0)?child((uint32_t)n3,b3):-1;
        if(i==0){
            loc.pos0++;                      // forced abstain, empty prefix
        } else if(ne>=0){                    // exact prefix present -> C>0
            double W=NW[ne];
            double wgt=(ce>=0)?(double)NW[ce]:0.0;
            int dsym=NC[ne]+(NE[ne]>0?1:0);
            if(wgt>0){ double cost=log2((W+1.0)/wgt); loc.cost+=cost; if(dsym==1)loc.det_c++; else loc.amb_h++; }
            else     { double cost=log2(W+1.0);       loc.cost+=cost; if(dsym==1)loc.det_w++; else loc.esc_w++; }
        } else {                             // exact abstain
            loc.ab++;
            bool found=false;
            if(!found && n2>=0 && c2>=0 && NW[c2]>0){ double W=NW[n2],w=NW[c2]; loc.var_cost+=log2((W+1.0)/w)+1.0; loc.var_hit++; found=true; }
            if(!found && n1>=0 && c1>=0 && NW[c1]>0){ double W=NW[n1],w=NW[c1]; loc.var_cost+=log2((W+1.0)/w)+1.0; loc.var_hit++; found=true; }
            if(!found && n3>=0 && c3>=0 && NW[c3]>0){ double W=NW[n3],w=NW[c3]; loc.var_cost+=log2((W+1.0)/w)+1.0; loc.var_hit++; found=true; }
            if(!found) loc.var_still++;
        }
        ne=ce; n1=c1; n2=c2; n3=c3;
    }
    // END event (prefix == full target)
    if(ne>=0){
        double W=NW[ne]; double wend=NE[ne]; int dsym=NC[ne]+(NE[ne]>0?1:0);
        if(wend>0){ double cost=log2((W+1.0)/wend); loc.e_cost+=cost; if(dsym==1)loc.e_det_c++; else loc.e_amb_h++; }
        else      { double cost=log2(W+1.0);        loc.e_cost+=cost; if(dsym==1)loc.e_det_w++; else loc.e_esc_w++; }
    } else loc.e_ab++;
    // classify construct
    int k;
    if(len>=9 && memcmp(s,"Category:",9)==0) k=1;
    else if(is_interwiki(s,len)) k=2;
    else k=0;
    int r = occOff<100000000ull?0 : occOff<400000000ull?1 : occOff<600000000ull?2 : 3;
    merge(G, loc);
    merge(REGCON[r][k], loc);
    insert(s,len);
}

// ---- reporting helpers ---------------------------------------------------------
static double bytesOf(double bits){ return bits/8.0; }

int main(int argc, char** argv){
    auto t0=chrono::steady_clock::now();
    const char* path = (argc>1)?argv[1]:"enwik9";
    FILE* f=fopen(path,"rb");
    if(!f){ fprintf(stderr,"cannot open %s\n",path); return 1; }
    fseek(f,0,SEEK_END); long long n=_ftelli64(f); fseek(f,0,SEEK_SET);
    fprintf(stderr,"file %s : %lld bytes\n", path, n);
    uint8_t* buf=(uint8_t*)malloc((size_t)n);
    if(!buf){ fprintf(stderr,"malloc failed\n"); return 1; }
    size_t got=0; while(got<(size_t)n){ size_t r=fread(buf+got,1,(size_t)n-got,f); if(!r)break; got+=r; }
    fclose(f);
    fprintf(stderr,"read %zu bytes\n", got);

    // init trie
    NW.reserve(50000000); NE.reserve(50000000); NC.reserve(50000000);
    H.init(1ull<<26);
    newnode(); // node 0 = root (empty prefix)

    uint64_t nTitles=0, nLinks=0;
    size_t N=got;
    size_t p=0;
    while(p<N){
        uint8_t ch=buf[p];
        if(ch=='<'){
            if(p+7<=N && memcmp(buf+p,"<title>",7)==0){
                size_t start=p+7;
                size_t q=start; size_t cap=start+320; if(cap>N)cap=N;
                bool nl=false;
                while(q<cap && !(buf[q]=='<' && q+8<=N && memcmp(buf+q,"</title>",8)==0)){ if(buf[q]=='\n') nl=true; q++; }
                if(q<cap && buf[q]=='<' && q+8<=N && memcmp(buf+q,"</title>",8)==0){
                    size_t len=q-start;
                    if(len>=1 && len<=256 && !nl){ insert(buf+start,len); nTitles++; }
                    p=q+8; continue;
                }
                p+=7; continue;
            }
            p++; continue;
        }
        if(ch=='['){
            if(p+1<N && buf[p+1]=='['){
                size_t start=p+2, q=start; bool invalid=false; size_t end=0;
                while(true){
                    if(q>=N){ invalid=true; break; }
                    uint8_t c=buf[q];
                    if(c=='\n'){ invalid=true; break; }
                    if(c=='|'){ end=q; break; }
                    if(c==']' && q+1<N && buf[q+1]==']'){ end=q; break; }
                    q++;
                    if(q-start>256){ invalid=true; break; }
                }
                if(!invalid){
                    size_t len=end-start;
                    if(len>=1 && len<=256){ process(buf+start,len,p); nLinks++; }
                }
                p+=2; continue; // continue scan after "[[" (finds nested links; outer not reprocessed)
            }
            p++; continue;
        }
        p++;
    }

    auto t1=chrono::steady_clock::now();
    double secs=chrono::duration<double>(t1-t0).count();

    // ---- derive strata ----
    Stats REG[4], CON[3], ALL=G;
    for(int r=0;r<4;r++) for(int k=0;k<3;k++){ merge(REG[r],REGCON[r][k]); merge(CON[k],REGCON[r][k]); }

    // memory
    PROCESS_MEMORY_COUNTERS pmc; memset(&pmc,0,sizeof(pmc));
    GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
    double peakMB = pmc.PeakWorkingSetSize/1048576.0;

    // ---- write report ----
    g_rep=fopen("e1_oracle_report.md","wb");

    auto nonab=[&](const Stats& s)->uint64_t{ return s.det_c+s.det_w+s.amb_h+s.esc_w; };

    auto blockTable=[&](const Stats& s){
        uint64_t na=nonab(s);
        double naive_all = 8.0*s.bytes;
        double oracle_na = s.cost;                       // bits, non-abstain only
        double naive_na  = 8.0*na;
        double oracle_all= s.cost + 8.0*(s.pos0+s.ab);   // abstain priced at naive 8 bpc
        L("| occurrences | %llu |\n",(unsigned long long)s.occ);
        L("| target bytes replayed | %llu |\n",(unsigned long long)s.bytes);
        L("| abstain bytes (pos0 %llu + later %llu) | %llu |\n",
             (unsigned long long)s.pos0,(unsigned long long)s.ab,(unsigned long long)(s.pos0+s.ab));
        L("| deterministic-correct | %llu |\n",(unsigned long long)s.det_c);
        L("| deterministic-wrong | %llu |\n",(unsigned long long)s.det_w);
        L("| ambiguous-hit | %llu |\n",(unsigned long long)s.amb_h);
        L("| escape/wrong | %llu |\n",(unsigned long long)s.esc_w);
        L("| non-abstain bytes | %llu |\n",(unsigned long long)na);
        L("| oracle log-loss (non-abstain) | %.0f B (%.4f bpc) |\n", bytesOf(oracle_na), na? oracle_na/na:0.0);
        L("| naive 8-bit baseline (non-abstain) | %.0f B |\n", bytesOf(naive_na));
        L("| oracle saving on non-abstain | %.0f B (%.1f%%) |\n", bytesOf(naive_na-oracle_na), naive_na>0?100.0*(naive_na-oracle_na)/naive_na:0.0);
        L("| oracle log-loss (ALL bytes, abstain@8bpc) | %.0f B (%.4f bpc) |\n", bytesOf(oracle_all), s.bytes? oracle_all/s.bytes:0.0);
        L("| naive 8-bit baseline (ALL bytes) | %.0f B |\n", bytesOf(naive_all));
        L("| oracle saving on ALL bytes | %.0f B (%.1f%%) |\n", bytesOf(naive_all-oracle_all), naive_all>0?100.0*(naive_all-oracle_all)/naive_all:0.0);
        // END events
        uint64_t e_na=s.e_det_c+s.e_det_w+s.e_amb_h+s.e_esc_w;
        L("| END events: det-correct %llu / det-wrong %llu / amb-hit %llu / escape %llu / abstain %llu | log-loss %.0f B |\n",
             (unsigned long long)s.e_det_c,(unsigned long long)s.e_det_w,(unsigned long long)s.e_amb_h,
             (unsigned long long)s.e_esc_w,(unsigned long long)s.e_ab, bytesOf(s.e_cost));
        (void)e_na;
    };

    L("# E1 oracle report -- exact prior-only prefix entity completion\n\n");
    L("**Evidence level: ORACLE** (decoder-mirrored, prior-only, exact raw bytes). Not an achieved compression result.\n\n");
    L("Input: `enwik9` (%lld bytes). Single online pass in file order. Population = exact raw strings from ",n);
    L("`<title>..</title>` contents + completed internal-link targets (bytes between `[[` and first `|`/`]]`; ");
    L("skipped if len 0, >256, or contains newline). A string enters the set the moment it completes; each link-target ");
    L("occurrence is replayed against the *prior* population before being inserted.\n\n");
    L("Escape model (PPM-style): for prefix P with total prior weight W, e=1/(W+1); a continuation symbol of weight w costs ");
    L("`-log2((1-e)*w/W) = log2((W+1)/w)`, an out-of-set next byte costs `-log2(e) = log2(W+1)`. ");
    L("DETERMINISTIC = exactly one distinct continuation symbol (child bytes + END) among prior strings with prefix P; else AMBIGUOUS. ");
    L("Position 0 (empty prefix) is a forced abstain, recorded separately. The END-of-string terminator is scored as one extra event per target.\n\n");

    L("## Population built (prior state at end of pass)\n\n");
    L("| titles inserted | %llu |\n|---|---:|\n",(unsigned long long)nTitles);
    L("| link-target occurrences processed | %llu |\n",(unsigned long long)nLinks);
    L("| total inserts (titles+targets) | %llu |\n",(unsigned long long)g_inserts);
    L("| distinct exact-raw strings | %llu |\n",(unsigned long long)g_distinct);
    L("| trie nodes | %llu |\n",(unsigned long long)NW.size());
    L("| trie edges (hash entries) | %llu |\n\n",(unsigned long long)H.cnt);

    L("## 1. Global outcome (all link-target constructs, whole file)\n\n");
    L("| metric | value |\n|---|---:|\n");
    blockTable(ALL);
    L("\n");

    L("## 2. Stratified by construct (whole file)\n\n");
    for(int k=0;k<3;k++){
        L("### %s\n\n| metric | value |\n|---|---:|\n",CONNAME[k]);
        blockTable(CON[k]);
        L("\n");
    }

    L("## 3. Stratified by file region (all constructs)\n\n");
    L("Compact per-region view (oracle vs naive on target bytes):\n\n");
    L("| region | occ | tgt bytes | abstain B | det-correct | ambig-hit | esc/wrong | oracle non-ab (B) | non-ab bpc | naive non-ab (B) | save %% | oracle ALL (B) | ALL bpc |\n");
    L("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n");
    for(int r=0;r<4;r++){
        const Stats& s=REG[r]; uint64_t na=nonab(s);
        double oracle_na=s.cost, naive_na=8.0*na, oracle_all=s.cost+8.0*(s.pos0+s.ab);
        L("| %s | %llu | %llu | %llu | %llu | %llu | %llu | %.0f | %.4f | %.0f | %.1f | %.0f | %.4f |\n",
             REGNAME[r],(unsigned long long)s.occ,(unsigned long long)s.bytes,(unsigned long long)(s.pos0+s.ab),
             (unsigned long long)s.det_c,(unsigned long long)s.amb_h,(unsigned long long)s.esc_w,
             bytesOf(oracle_na), na? oracle_na/na:0.0, bytesOf(naive_na),
             naive_na>0?100.0*(naive_na-oracle_na)/naive_na:0.0,
             bytesOf(oracle_all), s.bytes? oracle_all/s.bytes:0.0);
    }
    L("\nFull per-region tables:\n\n");
    for(int r=0;r<4;r++){
        L("### Region %s\n\n| metric | value |\n|---|---:|\n",REGNAME[r]);
        blockTable(REG[r]);
        L("\n");
    }

    L("## 4. Exact-raw vs variant-assisted coverage (normalization sensitivity)\n\n");
    L("The variant overlay is applied ONLY to bytes that ABSTAIN under exact-raw lookup (positions >=1 with no ");
    L("prior string on the exact prefix). It tries, in parallel, three normalized prefixes -- first-char case toggle, ");
    L("underscore<->space swap, and both -- and if one still predicts the (transformed) next byte, counts a VARIANT-HIT ");
    L("priced at its log-loss **+1 bit** (a variant-flag placeholder; recovering the exact byte spelling would cost more, ");
    L("so this is optimistic). This quantifies how much of the probe's normalized 'seen' coverage survives exact-byte prediction.\n\n");
    L("| stratum | later-abstain bytes | recovered by variant | still abstain | recovery %% | variant log-loss (B, incl +1b/byte) |\n");
    L("|---|---:|---:|---:|---:|---:|\n");
    {
        const Stats& s=ALL;
        L("| GLOBAL | %llu | %llu | %llu | %.1f | %.0f |\n",
             (unsigned long long)s.ab,(unsigned long long)s.var_hit,(unsigned long long)s.var_still,
             s.ab? 100.0*s.var_hit/s.ab:0.0, bytesOf(s.var_cost));
        for(int k=0;k<3;k++){ const Stats& c=CON[k];
            L("| %s | %llu | %llu | %llu | %.1f | %.0f |\n",CONNAME[k],
                 (unsigned long long)c.ab,(unsigned long long)c.var_hit,(unsigned long long)c.var_still,
                 c.ab?100.0*c.var_hit/c.ab:0.0, bytesOf(c.var_cost)); }
        for(int r=0;r<4;r++){ const Stats& c=REG[r];
            L("| region %s | %llu | %llu | %llu | %.1f | %.0f |\n",REGNAME[r],
                 (unsigned long long)c.ab,(unsigned long long)c.var_hit,(unsigned long long)c.var_still,
                 c.ab?100.0*c.var_hit/c.ab:0.0, bytesOf(c.var_cost)); }
    }
    {
        const Stats& s=ALL;
        double exact_cov = s.bytes>0 ? 100.0*(double)nonab(s)/(double)s.bytes : 0.0;
        double var_cov   = s.bytes>0 ? 100.0*(double)(nonab(s)+s.var_hit)/(double)s.bytes : 0.0;
        L("\n**Coverage of ALL target bytes:** exact-raw predicts %.2f%% of bytes (non-abstain / all); ",exact_cov);
        L("variant-assisted predicts %.2f%% (adds %.2f pts). Probe's normalized 'seen' occurrence rate was 76.38%%; ",var_cov,var_cov-exact_cov);
        L("exact-byte, prior-only *byte* prediction is materially lower because (i) position-0 and post-divergence bytes abstain, ");
        L("and (ii) exact raw bytes do not alias case/underscore variants.\n\n");
    }

    L("## 5. Resources\n\n");
    L("| metric | value |\n|---|---:|\n");
    L("| wall time (read+pass+report) | %.1f s |\n",secs);
    L("| peak working set | %.0f MB |\n",peakMB);
    L("| trie nodes | %llu (10 B each SoA) |\n",(unsigned long long)NW.size());
    L("| edge hash slots | %llu |\n",(unsigned long long)H.cap);
    L("| input buffer | %.0f MB |\n\n", n/1048576.0);

    L("## 6. Comparison to the first-100MB FULL audit (report sec.6)\n\n");
    {
        const Stats& r0all=REG[0];
        const Stats& r0plain=REGCON[0][0];
        double audit = 1776543.0 + 1178971.0; // LINK_TGT_NEW + LINK_TGT_SEEN, first 100 MB
        double r0all_na = bytesOf(r0all.cost);
        double r0all_all= bytesOf(r0all.cost + 8.0*(r0all.pos0+r0all.ab));
        double r0plain_na = bytesOf(r0plain.cost);
        L("- Audit `audit_e100m.txt`, first 100 MB, internal-link target classes: LINK_TGT_NEW 1,776,543 B + LINK_TGT_SEEN 1,178,971 B = **%.0f B**. ",audit);
        L("Those class bytes INCLUDE the `[[`/`]]` (and up to the `|`) delimiters and are the current CM's *attributed* cost; the oracle bytes below are target CONTENT bytes only (no delimiters) and are an ideal log-loss under the prior-only prefix model.\n");
        L("- Region 0-100 MB, ALL link constructs: oracle non-abstain log-loss = **%.0f B**; oracle ALL-bytes (abstain@8bpc) = **%.0f B** over %llu target bytes.\n",
             r0all_na, r0all_all, (unsigned long long)r0all.bytes);
        L("- Region 0-100 MB, PLAIN internal-link targets only (closest to LINK_TGT classes, still excludes delimiters): oracle non-abstain log-loss = **%.0f B** over %llu target bytes; abstain bytes %llu of %llu.\n",
             r0plain_na, (unsigned long long)r0plain.bytes,(unsigned long long)(r0plain.pos0+r0plain.ab),(unsigned long long)r0plain.bytes);
        L("- Interpretation: the oracle removes cost only from bytes it does NOT abstain on. The large abstain share (position-0 first bytes + every byte after a target's prefix leaves the population) is exactly the mass the audit still pays through the normal CM. The recoverable slice is the gap between the oracle non-abstain log-loss and 8 bpc on those same bytes.\n\n");
    }

    // ---- 5-line summary + verdict ----
    {
        const Stats& r0=REG[0];        // 0-100 MB   (comparable to audit_e100m)
        const Stats& rC=REG[2];        // 400-600 MB (census-dense; audit_e30m_census sits here)
        double r0_bpc = nonab(r0)? r0.cost/nonab(r0) : 0.0;
        double rC_bpc = nonab(rC)? rC.cost/nonab(rC) : 0.0;
        double all_bpc= nonab(ALL)? ALL.cost/nonab(ALL) : 0.0;
        double exact_cov = ALL.bytes? 100.0*(double)nonab(ALL)/(double)ALL.bytes : 0.0;
        double var_add   = ALL.bytes? 100.0*(double)ALL.var_hit/(double)ALL.bytes : 0.0;
        L("## 6b. Five-line summary vs audit-attributed link-target cost, and E1 gate verdict\n\n");
        L("1. Oracle non-abstain ideal log-loss (whole file) = **%.0f B at %.4f bpc** over %llu predicted target bytes (%.1f%% of all target bytes); abstain %llu bytes (pos0 + post-divergence).\n",
             bytesOf(ALL.cost), all_bpc, (unsigned long long)nonab(ALL), exact_cov, (unsigned long long)(ALL.pos0+ALL.ab));
        L("2. Region 0-100 MB oracle non-abstain = **%.4f bpc** vs the audit's `LINK_TGT_SEEN` = **1.1569 bpc** and `LINK_TGT_NEW` = 1.8582 bpc on the same slice: the oracle only *ties* the CM's already-attributed seen cost -- it does not undercut it.\n", r0_bpc);
        L("3. Region 400-600 MB (census-dense) oracle non-abstain = **%.4f bpc**, but the census-slice audit already codes `LINK_TGT_SEEN` at **0.1386 bpc** -- the current CM match/run-map is ~5x CHEAPER than this exact-prefix oracle exactly where repeated targets are densest.\n", rC_bpc);
        L("4. Normalization does not rescue coverage under exact bytes: the variant overlay recovers only %llu bytes (%.2f pts of all target bytes; %.1f%% of later-abstains) at +1 bit each -- the probe's 76.38%% normalized 'seen' is a normalization artifact, not exact-byte-recoverable mass.\n",
             (unsigned long long)ALL.var_hit, var_add, ALL.ab?100.0*ALL.var_hit/ALL.ab:0.0);
        L("5. **Recoverable cost after side information: MARGINAL, not the withdrawn multi-MB estimate.** The oracle is the UPPER BOUND on an E1 completion expert, yet its ideal log-loss (%.4f bpc; 0.72 bpc even in its best region) already sits AT or ABOVE the CM's attributed cost on the same bytes (1.1569 / 0.1386 bpc). The 100 MB 'saving vs 8 bpc' is illusory -- 8 bpc is not the operative baseline; the CM is. Headroom exists ONLY where the CM is not yet saturated: novel-target prefix completion (LINK_TGT_NEW 1.8582 bpc -> oracle ~1.12 bpc on covered bytes).\n\n", all_bpc);
        L("**Verdict (ORACLE): E1 survives only as a MARGINAL win; the large prize does NOT survive.** ");
        L("On the *seen-target* mass that motivated E1, no material recoverable cost survives exact-byte, prior-only prediction: the 16.4 MB / 1.1232 bpc oracle floor is not below the CM's attributed seen cost (1.1569 bpc early, 0.1386 bpc in the dense middle where the match/run-map is ~5x cheaper than the oracle), and ~20%% of target bytes abstain entirely. ");
        L("The only surviving headroom is prefix-completion of NOVEL targets (byte-coverage 79.9%% > occurrence-seen 76.38%% comes from shared prefixes), capped small -- consistent with the concurrently-measured E1 prototype's e100m -0.1641%% (~31 KB), which is ~2-3%% of the attributed link-target class cost, NOT the superseded 6-12 MB. ");
        L("This is the same set-membership signal the match model already carries, is subject to the ledger's 1 GB subsumption heuristic, and must still pay mixer-input + fingerprint metadata. Recommendation: the oracle does NOT justify scaling a standalone completion expert for a large prize; if E1 v1 is pursued it should be as a single confidence-gated mixer input judged at 1 GB on its own tiny margin, with expectation <=~0.2%% and real subsumption risk.\n\n");
    }

    L("## Method notes / caveats\n\n");
    L("- ORACLE, prior-only: state derives only from previously-completed titles/targets; each occurrence is scored before insertion.\n");
    L("- Exact raw bytes only; no entity decoding, no normalization in the primary pass. Category detected by literal `Category:` prefix; interwiki by the heuristic `^[a-z]{2,3}(-[a-z]+)*:` (matches the ledger's regex; a heuristic, may misfire).\n");
    L("- Heuristic byte-scan parser (like `field_probe.py`): `<title>` and `[[` matched literally in the byte stream; nested `[[` inside display text are processed as their own occurrences.\n");
    L("- The escape/log-loss is an ideal PPM-style cost, NOT the cost the real CM would pay after a mixer input, and NOT an entropy lower bound. Variant pricing (+1 bit) is a placeholder that under-counts exact-spelling recovery.\n");
    L("- END-of-target prediction is scored as a bonus event (not a target byte); it is reported separately and not folded into the byte bpc.\n");

    fclose(g_rep);

    // ---- stdout summary (ASCII only) ----
    uint64_t na=nonab(ALL);
    printf("=== E1 ORACLE (exact prior-only prefix) ===\n");
    printf("titles=%llu links=%llu distinct=%llu nodes=%llu edges=%llu\n",
        (unsigned long long)nTitles,(unsigned long long)nLinks,(unsigned long long)g_distinct,
        (unsigned long long)NW.size(),(unsigned long long)H.cnt);
    printf("target bytes=%llu  non-abstain=%llu  abstain(pos0=%llu,later=%llu)=%llu\n",
        (unsigned long long)ALL.bytes,(unsigned long long)na,(unsigned long long)ALL.pos0,
        (unsigned long long)ALL.ab,(unsigned long long)(ALL.pos0+ALL.ab));
    printf("det-correct=%llu det-wrong=%llu ambig-hit=%llu escape/wrong=%llu\n",
        (unsigned long long)ALL.det_c,(unsigned long long)ALL.det_w,(unsigned long long)ALL.amb_h,(unsigned long long)ALL.esc_w);
    printf("oracle non-abstain log-loss=%.0f B (%.4f bpc)  naive=%.0f B  save=%.1f%%\n",
        bytesOf(ALL.cost), na?ALL.cost/na:0.0, 8.0*na/8.0, na?100.0*(8.0*na-ALL.cost)/(8.0*na):0.0);
    printf("oracle ALL-bytes(abstain@8)=%.0f B (%.4f bpc)  naive=%.0f B  save=%.1f%%\n",
        bytesOf(ALL.cost+8.0*(ALL.pos0+ALL.ab)), ALL.bytes?(ALL.cost+8.0*(ALL.pos0+ALL.ab))/ALL.bytes:0.0,
        (double)ALL.bytes, ALL.bytes?100.0*(8.0*ALL.bytes-(ALL.cost+8.0*(ALL.pos0+ALL.ab)))/(8.0*ALL.bytes):0.0);
    printf("variant: later-abstain=%llu recovered=%llu (%.1f%%) still=%llu\n",
        (unsigned long long)ALL.ab,(unsigned long long)ALL.var_hit, ALL.ab?100.0*ALL.var_hit/ALL.ab:0.0,(unsigned long long)ALL.var_still);
    printf("time=%.1fs peakWS=%.0fMB\n", secs, peakMB);
    printf("wrote e1_oracle_report.md\n");
    return 0;
}