#!/usr/bin/env python3
"""
Rebuild every published figure for the Long Record of Study comparison.

    npm run long:build

This replaces align_sources.py, extract_passages.py and fuzzy_align.py, none of
which could produce what the page displayed. align_sources.py emitted a schema
with no page numbers, no body/reference split and no named/unnamed grouping;
extract_passages.py had no entry point; nothing at all produced passages.json.
The figures on the page therefore came from a process that was not in the
repository, over inputs that were not in the repository. This is that process,
written down.

WHAT IT READS
  sources.json          the corpus, and the only place a source is described
  document/*.pdf        the Record of Study, hash-identical to the published copy
  sources/*.txt         the source texts, extracted as sources.json records

WHAT IT WRITES, all into public/exhibits/long/
  density.json          headline figures, per-source results, per-page density
  passages.json         every matched run, with its real page and its context
  exclusions.json       every run that exists but is not counted, and why
  method/manifest.json  every file read, with SHA-256, provenance and word count

PAGE NUMBERS
  The Record of Study is extracted one PDF page at a time, so a matched run is
  attributed to the page the PDF actually puts it on. The previous data
  accumulated page numbers from word offsets over a stream whose page breaks did
  not match the document, and drifted: all sixty published passages were wrong,
  by +1 early and by +7 past page 200. Page attribution here cannot drift,
  because it is never computed.

  Every page carries its printed number as a running header, which pdftotext
  emits as the first line. Those headers were inside the analyzed text: they
  were counted as words, and they broke runs at every page boundary, so earlier
  longest-run figures are floors. The header is stripped before matching, and
  the strip is asserted rather than assumed.

WHAT IS COUNTED
  Chapters I to V only, PDF pages 18 to 182. A reference list agreeing with
  another reference list is not a finding, and the front matter is a title page.
  Matches outside the body are measured and published, in exclusions.json and on
  the page, but they are not in the headline. Segment boundaries are verified at
  runtime against the document's own section headings.
"""

import hashlib
import json
import re
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent
REPO = ROOT.parent.parent
OUT = REPO / "public" / "exhibits" / "long"
METHOD = OUT / "method"

WORD = re.compile(r"[^a-z0-9 ]")


# ---------------------------------------------------------------- tokenising

def normalise(s):
    """Lowercase, strip everything but letters, digits and spaces, then split."""
    return WORD.sub(" ", s.lower()).split()


def token_stream(raw_tokens, pages=None):
    """
    Map whitespace-separated raw tokens onto the normalized word stream.

    A raw token can yield more than one normalised word ("self-reports" gives
    two) or none at all ("---"), so the two streams are not the same length.
    idx[k] is the raw token that produced normalized word k, which is what lets
    a match found in normalized space be published as the original text.
    """
    norm, idx, page_of = [], [], []
    for i, tok in enumerate(raw_tokens):
        for w in normalise(tok):
            norm.append(w)
            idx.append(i)
            if pages is not None:
                page_of.append(pages[i])
    return norm, idx, page_of


def raw_slice(raw_tokens, idx, start, end):
    """The original text spanning a normalized half-open range."""
    if start >= len(idx):
        return ""
    a = idx[start]
    b = idx[min(end, len(idx)) - 1]
    return " ".join(raw_tokens[a : b + 1])


# ---------------------------------------------------------------- extraction

def pdf_pages(pdf, count=None):
    """Extract a PDF one page at a time, in reading order."""
    if count is None:
        info = subprocess.run(["pdfinfo", str(pdf)], capture_output=True, text=True).stdout
        m = re.search(r"^Pages:\s+(\d+)", info, re.M)
        count = int(m.group(1))
    out = []
    for p in range(1, count + 1):
        r = subprocess.run(
            ["pdftotext", "-f", str(p), "-l", str(p), str(pdf), "-"],
            capture_output=True, text=True,
        )
        out.append(r.stdout)
    return out


def strip_running_header(page_text, printed):
    """
    Remove the printed page number the document carries as a running header.

    Returns (text, stripped). Only the first non-empty line is considered, and
    only when it is exactly the number expected for that page, so a page that
    happens to open with a figure is never truncated.
    """
    lines = page_text.split("\n")
    for i, line in enumerate(lines):
        if not line.strip():
            continue
        if printed is not None and line.strip() == str(printed):
            return "\n".join(lines[:i] + lines[i + 1:]), True
        return page_text, False
    return page_text, False


def sha256(path):
    h = hashlib.sha256()
    h.update(Path(path).read_bytes())
    return h.hexdigest()


# ------------------------------------------------------------------ matching

def find_runs(doc, src, min_run):
    """
    Every maximal run of `min_run` or more consecutive words appearing in the
    same order in both, matched exactly after normalizing.

    Greedy and non-overlapping within a source: once a run is taken the walk
    resumes after it, so a source cannot be credited twice for the same words.
    Where several positions in the source match, the longest extension wins.
    """
    index = {}
    for j in range(len(src) - min_run + 1):
        index.setdefault(tuple(src[j : j + min_run]), []).append(j)

    runs = []
    i, n, m = 0, len(doc), len(src)
    while i <= n - min_run:
        cands = index.get(tuple(doc[i : i + min_run]))
        if not cands:
            i += 1
            continue
        best, best_j = 0, None
        for j in cands:
            k = min_run
            while i + k < n and j + k < m and doc[i + k] == src[j + k]:
                k += 1
            if k > best:
                best, best_j = k, j
        runs.append({"docAt": i, "len": best, "srcAt": best_j})
        i += best
    return runs


# ---------------------------------------------------------------------- main

def main():
    cfg = json.loads((ROOT / "sources.json").read_text())
    docCfg = cfg["document"]
    MIN_RUN = cfg["minimumRun"]
    offset = docCfg["printedPageOffset"]
    npages = docCfg["pages"]

    print(f"Reading {docCfg['pdf']} ({npages} pages)")
    pages = pdf_pages(ROOT / docCfg["pdf"], npages)
    if len(pages) != npages:
        sys.exit(f"expected {npages} pages, pdftotext gave {len(pages)}")

    # Strip the running header, and prove it was there.
    stripped = 0
    clean = []
    for p, text in enumerate(pages, 1):
        printed = p - offset if p > offset else None
        text, did = strip_running_header(text, printed)
        stripped += did
        clean.append(text)
    print(f"  running page-number header stripped from {stripped} of {npages} pages")
    if stripped < 150:
        sys.exit("header strip failed: the printed page numbers are still in the text")

    # One raw token stream for the whole document, each token knowing its page.
    raw_tokens, raw_pages = [], []
    for p, text in enumerate(clean, 1):
        for tok in text.split():
            raw_tokens.append(tok)
            raw_pages.append(p)
    doc, doc_idx, doc_page = token_stream(raw_tokens, raw_pages)
    print(f"  {len(doc):,} normalized words")

    # Verify the segment boundaries against the document's own headings rather
    # than trusting the numbers in sources.json.
    segs = docCfg["segments"]
    heads = {"body": "CHAPTER I", "references": "REFERENCES", "appendix": "APPENDIX A"}
    for s in segs:
        want = heads.get(s["name"])
        if not want:
            continue
        first = " ".join(clean[s["from"] - 1].split())[:60]
        if want not in first:
            sys.exit(f"segment {s['name']} should start at PDF {s['from']} with {want!r}, found {first!r}")
    print("  segment boundaries verified against the document's own headings")

    seg_of = {}
    for s in segs:
        for p in range(s["from"], s["to"] + 1):
            seg_of[p] = s["name"]
    counted_segs = {s["name"] for s in segs if s["counted"]}
    denom = sum(1 for k in range(len(doc)) if seg_of[doc_page[k]] in counted_segs)
    print(f"  counted denominator ({'/'.join(sorted(counted_segs))}): {denom:,} words")

    doc_lower = " ".join(doc)

    results, all_passages, all_excluded = {}, [], []
    manifest_files = [{
        "role": "document under examination",
        "cite": docCfg["cite"],
        "work": docCfg["work"],
        "file": Path(docCfg["pdf"]).name,
        "sha256": sha256(ROOT / docCfg["pdf"]),
        "bytes": (ROOT / docCfg["pdf"]).stat().st_size,
        "normalizedWords": len(doc),
        "publishedHere": docCfg["publishedHere"],
        "extraction": "pdftotext, reading order, one page at a time; running page-number header removed",
    }]

    for sc in cfg["sources"]:
        path = ROOT / sc["file"]
        if not path.exists():
            sys.exit(f"missing source text: {sc['file']}")
        text = path.read_text(encoding="utf8", errors="replace")
        src_raw = text.split()
        src, src_idx, _ = token_stream(src_raw)

        # Source page numbers, where a PDF survives to give them.
        src_page = None
        if sc.get("pdf") and (ROOT / sc["pdf"]).exists():
            sp = pdf_pages(ROOT / sc["pdf"])
            spr, sprp = [], []
            for p, t in enumerate(sp, 1):
                for tok in t.split():
                    spr.append(tok)
                    sprp.append(p)
            # Re-tokenising the per-page extraction can differ from the whole-file
            # one by stray whitespace, so only use it when the streams agree.
            s2, s2_idx, s2_page = token_stream(spr, sprp)
            if s2 == src:
                src_page = s2_page

        runs = find_runs(doc, src, MIN_RUN)

        # Is the source named anywhere in the document?
        #
        # A bare surname search is not enough to decide this and must never be
        # allowed to decide it silently. The document cites "Ramsey, Walker,
        # Shinn, and O'Neill (1989)", which is a different work by different
        # authors; a substring search moved the largest uncited source into the
        # named group on the strength of it. The decision is therefore reviewed
        # and recorded in sources.json, the search is still run, and a hit that
        # does not support the decision has to be explained in writing or the
        # build stops.
        hits = {n: doc_lower.count(n) for n in sc["names"]}
        named_total = sum(hits.values())
        declared = sc["namedInDocument"]
        if not declared and named_total and not sc.get("nameHitNote"):
            sys.exit(
                f"{sc['key']}: sources.json says it is never named, but the search "
                f"found {named_total} hit(s) {hits}. Add nameHitNote explaining them, "
                f"or change namedInDocument."
            )
        if declared and not named_total:
            sys.exit(
                f"{sc['key']}: sources.json says it is named, but the search found "
                f"nothing. Fix `names` or change namedInDocument."
            )

        by_seg, longest = {}, 0
        for r in runs:
            longest = max(longest, r["len"])
            for k in range(r["docAt"], r["docAt"] + r["len"]):
                by_seg[seg_of[doc_page[k]]] = by_seg.get(seg_of[doc_page[k]], 0) + 1

        counted_words = sum(v for k, v in by_seg.items() if k in counted_segs)
        counted_runs = 0

        for r in runs:
            page = doc_page[r["docAt"]]
            seg = seg_of[page]
            s, e = r["docAt"], r["docAt"] + r["len"]
            entry = {
                "source": sc["key"],
                "label": sc["label"],
                "len": r["len"],
                "page": page,
                "printedPage": page - offset if page > offset else None,
                "segment": seg,
                "counted": seg in counted_segs,
                "srcPage": src_page[r["srcAt"]] if src_page else None,
                "longPre": raw_slice(raw_tokens, doc_idx, max(0, s - 12), s),
                "longRun": raw_slice(raw_tokens, doc_idx, s, e),
                "longPost": raw_slice(raw_tokens, doc_idx, e, min(len(doc_idx), e + 12)),
                "srcPre": raw_slice(src_raw, src_idx, max(0, r["srcAt"] - 12), r["srcAt"]),
                "srcRun": raw_slice(src_raw, src_idx, r["srcAt"], r["srcAt"] + r["len"]),
                "srcPost": raw_slice(src_raw, src_idx, r["srcAt"] + r["len"],
                                     min(len(src_idx), r["srcAt"] + r["len"] + 12)),
            }
            if entry["counted"]:
                counted_runs += 1
                all_passages.append(entry)
            else:
                entry["reason"] = {
                    "references": "Reference-list material. Two documents citing the same literature are expected to agree.",
                    "appendix": "Appendix material, outside the chapters that make the argument.",
                    "front": "Front matter, not part of the body prose.",
                }[seg]
                all_excluded.append(entry)

        results[sc["key"]] = {
            "label": sc["label"],
            "cite": sc["cite"],
            "work": sc["work"],
            "url": sc.get("url"),
            "heldHere": sc.get("heldHere", False),
            "runs": counted_runs,
            "words": counted_words,
            "longest": longest,
            "bySegment": by_seg,
            "refWords": by_seg.get("references", 0),
            "refRuns": sum(1 for r in runs if seg_of[doc_page[r["docAt"]]] == "references"),
            "namedInDocument": declared,
            "status": sc["status"],
            "nameHits": hits,
            "nameHitNote": sc.get("nameHitNote"),
            "group": "named" if declared else "unnamed",
            "hasSourcePages": src_page is not None,
        }

        manifest_files.append({
            "role": "source",
            "key": sc["key"],
            "cite": sc["cite"],
            "work": sc["work"],
            "file": Path(sc["file"]).name,
            "sha256": sha256(path),
            "bytes": path.stat().st_size,
            "normalizedWords": len(src),
            "retrievedFrom": sc.get("url"),
            "retrievedOn": sc.get("retrievedOn"),
            "extraction": sc.get("extraction"),
            "sourcePdfSha256": sha256(ROOT / sc["pdf"]) if sc.get("pdf") and (ROOT / sc["pdf"]).exists() else None,
            "publicCopyAvailable": bool(sc.get("url")),
        })
        print(f"  {sc['key']:<14} runs={counted_runs:<4} words={counted_words:<6} longest={longest:<4} {results[sc['key']]['group']}")

    # Deduplicated distinct words, which is what the headline reports. A passage
    # can match more than one source, so per-source totals add to more.
    covered, unnamed_cov = set(), set()
    for sc in cfg["sources"]:
        text = (ROOT / sc["file"]).read_text(encoding="utf8", errors="replace")
        src, _, _ = token_stream(text.split())
        for r in find_runs(doc, src, MIN_RUN):
            for k in range(r["docAt"], r["docAt"] + r["len"]):
                if seg_of[doc_page[k]] in counted_segs:
                    covered.add(k)
                    if results[sc["key"]]["group"] == "unnamed":
                        unnamed_cov.add(k)

    body_matched = len(covered)
    unnamed_words = len(unnamed_cov)
    named_only = body_matched - unnamed_words
    ref_matched = sum(r["refWords"] for r in results.values())

    # Per-page density.
    page_words, page_matched, page_top = {}, {}, {}
    per_page_src = {}
    for k in range(len(doc)):
        p = doc_page[k]
        page_words[p] = page_words.get(p, 0) + 1
    for sc in cfg["sources"]:
        text = (ROOT / sc["file"]).read_text(encoding="utf8", errors="replace")
        src, _, _ = token_stream(text.split())
        for r in find_runs(doc, src, MIN_RUN):
            for k in range(r["docAt"], r["docAt"] + r["len"]):
                if seg_of[doc_page[k]] not in counted_segs:
                    continue
                per_page_src.setdefault(doc_page[k], {})
                per_page_src[doc_page[k]][sc["key"]] = per_page_src[doc_page[k]].get(sc["key"], 0) + 1
    for k in covered:
        p = doc_page[k]
        page_matched[p] = page_matched.get(p, 0) + 1

    density = []
    for p in range(1, npages + 1):
        w = page_words.get(p, 0)
        m = page_matched.get(p, 0)
        top = None
        if per_page_src.get(p):
            top = max(per_page_src[p].items(), key=lambda kv: kv[1])[0]
        density.append({
            "page": p,
            "printedPage": p - offset if p > offset else None,
            "segment": seg_of[p],
            "words": w,
            "matched": m,
            "pct": round(100 * m / w) if w else 0,
            "top": top,
        })

    pages_with_match = sum(1 for d in density if d["matched"] > 0)
    longest_run = max((r["longest"] for r in results.values()), default=0)

    out = {
        "generated": subprocess.run(["date", "-u", "+%Y-%m-%d"], capture_output=True, text=True).stdout.strip(),
        "docWords": len(doc),
        "bodyWords": denom,
        "bodyMatched": body_matched,
        "bodyPct": round(100 * body_matched / denom, 1),
        "refMatched": ref_matched,
        "unnamedWords": unnamed_words,
        "namedOnlyWords": named_only,
        "pagesWithMatch": pages_with_match,
        "pagesTotal": npages,
        "longestRun": longest_run,
        "countedSources": len(cfg["sources"]),
        "sources": results,
        "density": density,
        # Kept because the page reads them; same values, older names.
        "covered": body_matched,
        "pct": round(100 * body_matched / denom, 1),
    }

    all_passages.sort(key=lambda p: -p["len"])
    all_excluded.sort(key=lambda p: -p["len"])
    for i, p in enumerate(all_passages, 1):
        p["id"] = f"passage-{i:02d}"
    for i, p in enumerate(all_excluded, 1):
        p["id"] = f"excluded-{i:02d}"

    OUT.mkdir(parents=True, exist_ok=True)
    METHOD.mkdir(parents=True, exist_ok=True)
    (OUT / "density.json").write_text(json.dumps(out, indent=1))
    (OUT / "passages.json").write_text(json.dumps(all_passages, indent=1))
    (OUT / "exclusions.json").write_text(json.dumps(all_excluded, indent=1))

    manifest = {
        "generated": out["generated"],
        "method": {
            "script": "build.py",
            "command": "npm run long:build",
            "minimumRun": MIN_RUN,
            "normalization": "lowercase, strip everything but letters digits and spaces, split on whitespace",
            "matching": "exact after normalizing; one changed word ends a run",
            "extraction": "pdftotext in reading order, one page at a time. Page-layout order interleaves columns and destroys long runs; on the Ripple Effects monograph it reduces the longest run from 155 words to 44.",
            "runningHeader": f"The printed page number is a running header on every page and was removed before matching, on {stripped} of {npages} pages. Left in, it is counted as a word and ends any run that crosses a page boundary.",
            "denominator": f"{denom:,} words, Chapters I to V, PDF pages 18 to 182. Front matter, the reference list and the appendices are outside both the numerator and the denominator.",
            "excluded": [
                f"Runs shorter than {MIN_RUN} words.",
                "Matches in the reference list, the appendices and the front matter. These are measured and published in exclusions.json, and are not in the headline.",
                "McNeil, J. D., Curriculum: A Comprehensive Introduction, which is documented on the page but not counted, because the book is not held here and cannot be measured.",
            ],
            "knownLimits": [
                "Figures are a floor. Paraphrase, synonym substitution and reordering are invisible to exact matching.",
                "Sources reachable only through ProQuest, and books, are not covered.",
            ],
        },
        "files": manifest_files,
        "results": {k: out[k] for k in (
            "docWords", "bodyWords", "bodyMatched", "bodyPct", "refMatched",
            "unnamedWords", "namedOnlyWords", "pagesWithMatch", "pagesTotal",
            "longestRun", "countedSources")},
    }
    (METHOD / "manifest.json").write_text(json.dumps(manifest, indent=1))

    # Publish the script and its configuration next to the results. Copying them
    # here rather than maintaining a second copy is the point: the published
    # script is the one that ran, and cannot drift from it. The previous
    # published script emitted a different schema from the data beside it.
    (METHOD / "build.py").write_text((ROOT / "build.py").read_text())
    (METHOD / "sources.json").write_text((ROOT / "sources.json").read_text())

    print()
    print(f"  body {denom:,} words | matched {body_matched:,} ({out['bodyPct']}%)")
    print(f"  unnamed {unnamed_words:,} | named-only {named_only:,} | reference list {ref_matched:,}")
    print(f"  {pages_with_match} of {npages} pages carry a counted match | longest run {longest_run}")
    print(f"  {len(all_passages)} counted passages, {len(all_excluded)} excluded")


if __name__ == "__main__":
    main()