_tmp_debug_stats.py

import json
import re

fp = "debug_sent_eval.jsonl"
span_re = re.compile(r"<(AI|HUMAN)>(.*?)</\1>", re.I | re.S)
all_tag_re = re.compile(r"<\s*/?\s*([A-Za-z\u4e00-\u9fff0-9_\-]+)[^>]*>")


def split_simple(t):
    t = str(t).replace("\r", "").replace("\n", "").strip()
    if not t:
        return []
    arr = []
    start = 0
    for idx, ch in enumerate(t):
        if ch in "。！？!?":
            seg = t[start: idx + 1].strip()
            if seg:
                arr.append(seg)
            start = idx + 1
        elif ch == ".":
            if idx > 0 and idx + 1 < len(t) and t[idx - 1].isdigit() and t[idx + 1].isdigit():
                continue
            seg = t[start: idx + 1].strip()
            if seg:
                arr.append(seg)
            start = idx + 1
    tail = t[start:].strip()
    if tail:
        arr.append(tail)
    return arr


samples = 0
unknown_tag_samples = 0
unknown_tag_total = 0
exact_overlap_rates = []

with open(fp, "r", encoding="utf-8") as f:
    for line in f:
        d = json.loads(line)
        samples += 1

        pred = d.get("pred_tagged", "")
        tags = all_tag_re.findall(pred)
        bad = [x for x in tags if x.upper() not in {"AI", "HUMAN"}]
        if bad:
            unknown_tag_samples += 1
            unknown_tag_total += len(bad)

        gold_pairs = span_re.findall(d.get("eval_gold_tagged", ""))
        pred_pairs = span_re.findall(pred)

        gold_s = []
        for _, txt in gold_pairs:
            gold_s.extend(split_simple(txt))

        pred_s = []
        for _, txt in pred_pairs:
            pred_s.extend(split_simple(txt))

        gs = set(x.strip() for x in gold_s if x.strip())
        ps = set(x.strip() for x in pred_s if x.strip())

        if gs:
            exact_overlap_rates.append(len(gs & ps) / len(gs))

print("samples", samples)
print("unknown_tag_samples", unknown_tag_samples, f"({unknown_tag_samples / samples:.1%})")
print("unknown_tag_total", unknown_tag_total)
print("mean_exact_overlap", sum(exact_overlap_rates) / len(exact_overlap_rates))
print("min_exact_overlap", min(exact_overlap_rates), "max_exact_overlap", max(exact_overlap_rates))