#!/usr/bin/env python
"""content-c6281-000: the word census. For each primary document in sources/, extract the text, count every hit of
agent / agents / agentic / autonomous* and print each 'agent' hit in context with its page (PDF) or its position (HTML),
then pull the document's own definition of "AI system" / "artificial intelligence" by anchor. Every number in
research.md comes out of this script. Run from anywhere; it reads and writes beside itself.

    python agent_word_census_c6281.py  >  agent_word_census_c6281.out.txt
"""
import html as html_mod
import json
import re
import sys
from pathlib import Path

import fitz  # PyMuPDF

HERE = Path(__file__).resolve().parent
SRC = HERE / "sources"

DOCS = [
    # key, file, label, definition anchor regex (case-insensitive), chars to keep after the anchor
    ("eu_ai_act", "eu_ai_act_2024_1689.pdf", "Regulation (EU) 2024/1689 (AI Act), OJ L 12.7.2024",
     r"\(1\)\s*['‘]AI system['’]\s*means", 700),
    ("nist_rmf", "nist_ai_100_1.pdf", "NIST AI 100-1, AI RMF 1.0 (Jan 2023)",
     r"an AI system to be an engineered or machine-based system", 700),
    ("omb_m2521", "omb_m_25_21.pdf", "OMB M-25-21 (Apr 3, 2025)",
     r"term\s*['‘“]artificial intelligence['’”]\s*(?:or\s*['‘“]AI['’”]\s*)?(?:has the meaning|means)", 700),
    ("usc_9401", "usc_15_9401.html", "15 U.S.C. 9401 (LII)",
     r"The term\s*['‘“]artificial intelligence['’”]\s*means", 700),
    ("colorado", "colorado_sb24_205.pdf", "Colorado SB 24-205 (signed 2024)",
     r"['\"‘“]ARTIFICIAL INTELLIGENCE SYSTEM['\"’”]\s*MEANS", 600),
    ("california_sb53", "california_sb53.html", "California SB 53 (2025)",
     r"['‘“]Artificial intelligence model['’”]\s*means", 600),
    ("texas_hb149", "texas_hb149.pdf", "Texas HB 149 (TRAIGA, 2025)",
     r"['\"‘“]Artificial intelligence system['\"’”]\s*means", 600),
    ("utah_sb149", "utah_sb149_2024_enrolled.pdf", "Utah SB 149 (AI Policy Act, 2024, enrolled)",
     r"['\"‘“]Generative artificial intelligence['\"’”]\s*means", 600),
    ("ec_guidelines", "ec_guidelines_ai_system_definition_c2025_924.pdf",
     "Commission Guidelines on the definition of an AI system, C(2025) 5053 final (29.7.2025)", r"seven main elements", 400),
    ("ndaa_238g", "plaw_115_232.htm", "Pub. L. 115-232 sec. 238(g), 132 Stat. 1697-98 (the definition OMB M-25-21 adopts)",
     r"Artificial Intelligence Defined\.--In this section", 1000),
    ("iso_22989", "iso_iec_22989_2022_iteh_sample.pdf", "ISO/IEC 22989:2022 preview (iTeh sample), clause 3.1",
     r"3\.1\.1 AI agent", 200),
    ("eo14179", "eo14179.txt", "EO 14179 (Jan 23, 2025), 90 FR 8741", r"NEVER-MATCHES", 0),
    ("eo14277", "eo14277.txt", "EO 14277 (Apr 23, 2025)", r"NEVER-MATCHES", 0),
    ("eo14363", "eo14363.txt", "EO 14363 Launching the Genesis Mission (Nov 24, 2025), 90 FR 55035", r"NEVER-MATCHES", 0),
    ("eo14365", "eo14365.txt", "EO 14365 National Policy Framework for AI (Dec 11, 2025), 90 FR 58499", r"NEVER-MATCHES", 0),
    ("eo14409", "eo14409.txt", "EO 14409 Promoting Advanced AI Innovation and Security (Jun 2, 2026)", r"NEVER-MATCHES", 0),
    ("fr_nist_rfi", "fr_nist_rfi_agents.txt", "NIST/CAISI RFI, Security Considerations for AI Agents, 91 FR 699 (Jan 8, 2026)",
     r"AI agent systems are capable of planning", 600),
    ("fr_hhs_onc", "fr_hhs_onc.txt", "HHS/ASTP-ONC proposed rule, 90 FR 61005-06 (Dec 29, 2025)",
     r"Autonomous artificial intelligence systems that are designed to execute", 700),
]
AUTONOMY_KWIC = {"eu_ai_act", "nist_rmf", "colorado", "texas_hb149", "california_sb53"}

WORDS = {
    "agent": re.compile(r"\bagents?\b", re.I),
    "agentic": re.compile(r"\bagentic\b", re.I),
    "autonom*": re.compile(r"\bautonom\w*", re.I),
    "AI system": re.compile(r"\bAI systems?\b|\bartificial intelligence systems?\b", re.I),
}


def pdf_pages(path):
    doc = fitz.open(path)
    return [(i + 1, p.get_text("text")) for i, p in enumerate(doc)]


def html_text(path):
    raw = path.read_text(encoding="utf-8", errors="replace")
    if path.suffix == ".txt":                       # Federal Register full-text dumps are already plain text
        return [(0, raw)]
    raw = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", raw)
    txt = html_mod.unescape(re.sub(r"<[^>]+>", " ", raw))
    return [(0, txt)]


def norm(s):
    return re.sub(r"\s+", " ", s)


def main():
    out = {}
    for key, fname, label, anchor, keep in DOCS:
        path = SRC / fname
        if not path.exists():
            print("MISSING", key, fname)
            continue
        pages = pdf_pages(path) if fname.endswith(".pdf") else html_text(path)
        full = norm(" ".join(t for _, t in pages))
        counts = {w: len(rx.findall(full)) for w, rx in WORDS.items()}
        hits = []
        for pno, text in pages:
            t = norm(text)
            for m in WORDS["agent"].finditer(t):
                hits.append({"page": pno, "kwic": t[max(0, m.start() - 110):m.end() + 110]})
        m = re.search(anchor, full, re.I)
        definition = full[m.start():m.start() + keep] if m else None
        out[key] = {"label": label, "file": fname, "pages": len(pages) if fname.endswith(".pdf") else None,
                    "chars": len(full), "counts": counts, "agent_hits": hits, "definition": definition}
        print("=" * 100)
        print(f"{key}  {label}  [{fname}; {out[key]['pages']} pages; {len(full):,} chars]")
        print("  counts:", counts)
        for h in hits:
            print(f"  agent @p{h['page']}: …{h['kwic']}…")
        if key in AUTONOMY_KWIC:
            for pno, text in pages:
                tt = norm(text)
                for m in WORDS["autonom*"].finditer(tt):
                    print(f"  autonom* @p{pno}: ...{tt[max(0, m.start() - 120):m.end() + 120]}...")
        print("  DEFINITION:", definition if definition else "ANCHOR NOT FOUND")
    (HERE / "agent_word_census_c6281.json").write_text(json.dumps(out, indent=1, ensure_ascii=False), encoding="utf-8")
    print("=" * 100)
    print("wrote agent_word_census_c6281.json")


if __name__ == "__main__":
    main()