"""Every figure and every quotation in "The Pentagon's Four $200 Million AI Deals Have Obligated $45.5 Million.
USAspending Shows $0 of It.", recomputed from the saved primaries and asserted.

Inputs (beside this script): sources/ (every primary as bravo fetched it on 2026-09-21): the FPDS-NG ATOM pulls
(fpds_hq0883_2025.xml, fpds_hq0883_2026.xml, fpds_openai_free.xml, fpds_anthropic_free.xml, fpds_xai_free.xml), the
USAspending API responses (usaspending_vendor_scan.json, usaspending_piid_lookup.json), the USAspending "About the Data"
PDF, the FPDS DoD data-availability notice, GAO-25-107546, the xAI GSA agreement PDF, and the trade-press pages that
carry the DoD, CDAO and OpenAI announcement sentences.

The script re-parses the FPDS entries with the pull script's own field regexes, de-duplicates on (PIID, modification
number), re-totals every HQ0883 agreement, and asserts each amount, date, count, share and quotation the essay carries.
A figure that drifts from its source fails the run. It also records the one correction to the research draft: that
draft's Table 1 carried xAI's agreement at $1,999,998 (two later obligations, $2,500,000.42 on 2025-12-18 and
$7,500,000.00 on 2026-05-15, were in the pull's own tabulation and not in the table), so the four-agreement total the
essay prints is $45,540,443.89, not $35,540,443.47.

Run: python figures_c6623.py            (reads sources/ next to it; no network)
"""
import html as html_mod
import json
import re
import sys
from pathlib import Path

import fitz  # PyMuPDF

HERE = Path(__file__).resolve().parent
SRC = HERE / "sources"
FAILS = []


def check(cond, label):
    print(("  ok   " if cond else "  FAIL ") + label)
    if not cond:
        FAILS.append(label)


def norm(s):
    return re.sub(r"\s+", " ", s)


def html_text(name):
    t = (SRC / name).read_text(encoding="utf-8", errors="replace")
    t = re.sub(r"<script.*?</script>|<style.*?</style>", " ", t, flags=re.S)
    t = re.sub(r"<[^>]+>", " ", t)
    return norm(html_mod.unescape(t))


def pdf_pages(name):
    return [norm(p.get_text("text")) for p in fitz.open(SRC / name)]


# ── 1. FPDS-NG: re-parse the pulls with the pull script's own regexes, de-duplicate on (PIID, mod) ──────────────
def g(e, p):
    m = re.search(p, e, re.S)
    return html_mod.unescape(m.group(1).strip()) if m else ""


rows = {}
for f in ("fpds_hq0883_2025.xml", "fpds_hq0883_2026.xml", "fpds_openai_free.xml", "fpds_anthropic_free.xml", "fpds_xai_free.xml"):
    b = (SRC / f).read_text(encoding="utf-8", errors="replace")
    for e in re.findall(r"<entry>(.*?)</entry>", b, re.S):
        piid = g(e, r"<ns1:PIID>([^<]+)")
        mod = g(e, r"<ns1:modNumber>([^<]+)") or "0"
        key = (piid, mod)
        if key in rows:
            continue
        rows[key] = {
            "vendor": g(e, r"<ns1:vendorName>([^<]+)"), "piid": piid, "mod": mod,
            "signed": g(e, r"<ns1:signedDate>([^<]+)")[:10],
            "obligated": float(g(e, r"<ns1:obligatedAmount>([^<]+)") or 0),
            "agency": g(e, r'<ns1:contractingOfficeAgencyID[^>]*name="([^"]+)"'),
            "office": g(e, r'<ns1:contractingOfficeID[^>]*name="([^"]+)"'),
            "type": g(e, r'<ns1:contractActionType[^>]*description="([^"]+)"'),
            "reason": g(e, r'<ns1:reasonForModification[^>]*description="([^"]+)"'),
            "desc": g(e, r"<ns1:descriptionOfContractRequirement>([^<]+)"),
            "completion": g(e, r"<ns1:currentCompletionDate>([^<]+)")[:10],
        }
print("FPDS entries parsed (de-duplicated on PIID + modification):", len(rows))


def actions(piid):
    return sorted((r for r in rows.values() if r["piid"] == piid), key=lambda r: (r["signed"], r["mod"]))


def total(piid):
    return round(sum(r["obligated"] for r in actions(piid)), 2)


FOUR = {"HQ08832590012": ("OPENAI PUBLIC SECTOR, LLC", "OPEN AI FRONTIER AI PROTOTYPE"),
        "HQ08832590013": ("GOOGLE PUBLIC SECTOR LLC", "FRONTIER AI GOOGLE PROTOTYPE"),
        "HQ08832590014": ("ANTHROPIC, PBC", "FRONTIER AI ANTHROPIC PROTOTYPE"),
        "HQ08832590015": ("AIQ PHASE LLC", "FRONTIER AI AIQ PROTOTYPE")}

print("\n== the four frontier agreements ==")
for piid, (vendor, desc0) in FOUR.items():
    a = actions(piid)
    check(a and a[0]["vendor"] == vendor, f"{piid} vendor {vendor}")
    check(a and a[0]["desc"] == desc0, f"{piid} base description {desc0!r}")
    check(all(r["type"] == "OTHER TRANSACTION AGREEMENT" for r in a), f"{piid} every action is OTHER TRANSACTION AGREEMENT")
    check(a and a[0]["obligated"] == 1999998.00, f"{piid} obligated at award $1,999,998.00")
    check(a and "CHIEF DIGITAL" in a[0]["office"].upper() and "SECRETARY OF DEFENSE" in a[0]["agency"].upper(),
          f"{piid} contracting office is the CDAO under the Immediate Office of the Secretary of Defense")

# OpenAI: base 2025-06-16; six actions; one $1.00 change order on 2026-02-06 moving completion to 2027-06-15
a = actions("HQ08832590012")
check(a[0]["signed"] == "2025-06-16", "OpenAI base signed 2025-06-16")
check(len(a) == 6, f"OpenAI: 6 actions in the record (found {len(a)})")
one = [r for r in a if r["obligated"] == 1.00]
check(len(one) == 1 and one[0]["signed"] == "2026-02-06" and one[0]["reason"] == "CHANGE ORDER", "OpenAI: the one-dollar change order is dated 2026-02-06")
check(a[0]["completion"] == "2026-06-15" and one[0]["completion"] == "2027-06-15", "OpenAI: completion moved from 2026-06-15 to 2027-06-15 on that action")
check(total("HQ08832590012") == 1999999.00, "OpenAI total obligated $1,999,999.00")

# Google: base 2025-07-14 and five later obligations
a = actions("HQ08832590013")
check(a[0]["signed"] == "2025-07-14", "Google base signed 2025-07-14")
later = [(r["signed"], r["obligated"]) for r in a[1:]]
check(later == [("2025-08-14", 3000000.00), ("2025-11-17", 0.47), ("2025-12-23", 1500000.00), ("2026-03-24", 3040450.00), ("2026-04-27", 20000000.00)],
      f"Google: the five later obligations (found {later})")
check(total("HQ08832590013") == 29540448.47, "Google total obligated $29,540,448.47")
check(a[-1]["completion"] == "2027-07-14", "Google completion 2027-07-14")

# Anthropic: one action only
a = actions("HQ08832590014")
check(a[0]["signed"] == "2025-07-14" and len(a) == 1, f"Anthropic: the award and nothing after it (found {len(a)} action(s))")
check(total("HQ08832590014") == 1999998.00, "Anthropic total obligated $1,999,998.00")
check(a[0]["completion"] == "2027-07-14", "Anthropic completion 2027-07-14")

# xAI (AIQ Phase LLC): base, an administrative $0, $2,500,000.42 on 2025-12-18, a $0 change order, $7,500,000.00 on 2026-05-15 described SPACEX PROTOTYPE
a = actions("HQ08832590015")
check(a[0]["signed"] == "2025-07-14" and len(a) == 5, f"xAI: five actions (found {len(a)})")
amts = [(r["signed"], r["obligated"], r["desc"]) for r in a]
check(amts[2] == ("2025-12-18", 2500000.42, "AIQ FRONTIER AI"), f"xAI: $2,500,000.42 on 2025-12-18 (found {amts[2]})")
check(amts[4] == ("2026-05-15", 7500000.00, "SPACEX PROTOTYPE"), f"xAI: $7,500,000.00 on 2026-05-15 described SPACEX PROTOTYPE (found {amts[4]})")
check(total("HQ08832590015") == 11999998.42, "xAI total obligated $11,999,998.42")

four_total = round(sum(total(p) for p in FOUR), 2)
check(four_total == 45540443.89, f"four together $45,540,443.89 (found {four_total:,.2f})")
four_actions = sum(len(actions(p)) for p in ("HQ08832590012", "HQ08832590013", "HQ08832590014", "HQ08832590015"))
check(four_actions == 18, f"the four frontier agreements: eighteen actions in the record, awards included (found {four_actions}) [packaging, scout bv1, 2026-09-22]")
check(round(100 * four_total / 8e8, 2) == 5.69, f"share of the $800,000,000 of ceilings 5.69% (found {100 * four_total / 8e8:.2f}%)")
check(round(100 * total('HQ08832590013') / four_total, 1) == 64.9, f"Google's share of the four 64.9% (found {100 * total('HQ08832590013') / four_total:.1f}%)")
check(sum(actions(p)[0]["obligated"] for p in FOUR) == 7999992.00, "day-one obligations $7,999,992 (four times $1,999,998)")
check(four_total - 35540443.47 == 10000000.42 or abs(four_total - 35540443.47 - 10000000.42) < 0.01,
      "the research draft's $35,540,443.47 is short by exactly xAI's two later obligations ($10,000,000.42)")

# ── 2. the same office's other agreements, same vehicle ────────────────────────────────────────────────────────
print("\n== the same office, same vehicle ==")
hq = sorted({r["piid"] for r in rows.values() if r["piid"].startswith("HQ0883")})
check(len(hq) == 33, f"33 distinct HQ0883 agreements in the two PIID sweeps plus the vendor pulls (found {len(hq)})")
expect = {"HQ0883269E001": ("KNOSSOS GROUP GLOBAL, LLC", 123468267.25), "HQ08832590001": ("ANDURIL INDUSTRIES, INC.", 85921414.78),
          "HQ08832590005": ("DATABRICKS FEDERAL LLC", 74997002.33), "HQ0883269E011": ("PALANTIR USG INC", 60999999.96),
          "HQ0883269E004": ("DATABRICKS FEDERAL LLC", 16870233.05), "HQ0883269E002": ("GAME PLAN TECH LLC", 11119208.00),
          "HQ0883269E016": ("GOOGLE PUBLIC SECTOR LLC", 4999999.00), "HQ0883269E018": ("ORACLE AMERICA, INC", 4821654.00),
          "HQ0883269E017": ("MICROSOFT CORPORATION", 3960000.00), "HQ0883269E015": ("AMAZON WEB SERVICES, INC.", 2230784.00),
          "HQ0883259E004": ("PALANTIR USG INC", 1250000.00)}
for piid, (vendor, amt) in expect.items():
    a = actions(piid)
    check(a and a[0]["vendor"] == vendor and total(piid) == amt, f"{piid} {vendor} ${amt:,.2f} (found {total(piid):,.2f})")
kn = actions("HQ0883269E001")
check([(r["signed"], r["obligated"]) for r in kn] == [("2025-11-14", 200000.00), ("2025-12-03", 72800000.00), ("2026-01-30", 4835868.00), ("2026-05-29", 45632399.25)],
      "Knossos: four actions, $200,000 then $72.8 million then $4.8 million then $45.6 million")
check({r["desc"] for r in kn} == {"HOMELAND INNOVATION PROTOTYPE", "OCEANS PROTOTYPE"}, "Knossos descriptions HOMELAND INNOVATION PROTOTYPE / OCEANS PROTOTYPE")
pal = actions("HQ0883269E011")
check(pal[0]["desc"] == "WAR MACHINE" and pal[0]["signed"] == "2026-04-28" and pal[1]["obligated"] == 55999999.96 and pal[1]["signed"] == "2026-06-05",
      "Palantir WAR MACHINE: $5,000,000 on 2026-04-28 and $55,999,999.96 on 2026-06-05")
check(actions("HQ08832590001")[0]["desc"] == "ANDURIL TDS/TDM PRODUCTION AGREEMENT" and actions("HQ08832590001")[0]["signed"] == "2024-12-02", "Anduril: TDS/TDM production agreement from 2024-12-02")
ranked = sorted(hq, key=lambda p: -total(p))
check(ranked[:4] == ["HQ0883269E001", "HQ08832590001", "HQ08832590005", "HQ0883269E011"], f"the four largest CDAO agreements are Knossos, Anduril, Databricks, Palantir (found {ranked[:4]})")
check(all(total(p) < four_total for p in ranked[4:]) and total("HQ0883269E011") > four_total, "the four frontier agreements together rank behind those four")
check(round(total('HQ0883269E001') / four_total, 1) == 2.7, f"Knossos alone is 2.7 times the four frontier agreements combined (found {total('HQ0883269E001') / four_total:.2f})")
newest = max(r["signed"] for r in rows.values() if r["piid"].startswith("HQ0883"))
check(newest == "2026-06-18", f"newest signedDate across the HQ0883 entries 2026-06-18 (found {newest})")
n_hq = sum(1 for r in rows.values() if r["piid"].startswith("HQ0883"))
check(n_hq == 92, f"92 CDAO actions retrieved across the two PIID sweeps (found {n_hq})")
# the essay's derived sentences
check(round(545578476.94 / four_total) == 12, f"Palantir's Maven task order is twelve times the four frontier agreements (found {545578476.94 / four_total:.2f})")
from datetime import date
check((date(2025, 12, 3) - date(2025, 11, 14)).days == 19 and round(kn[0]["obligated"] + kn[1]["obligated"]) == 73000000, "Knossos: $200,000 to $73 million in nineteen days")
check(round((total("HQ08832590005") + total("HQ0883269E004")) / 1e6, 1) == 91.9, "Databricks: two agreements worth $91.9 million between them")
check((date(2025, 7, 14) - date(2024, 12, 2)).days // 30 == 7, "Anduril's agreement predates the frontier awards by seven months")
cloud = {p: actions(p) for p in ("HQ0883269E016", "HQ0883269E018", "HQ0883269E017", "HQ0883269E015")}
check(all(len(a) == 2 and "2026-05-21" <= a[0]["signed"] <= "2026-05-31" and "2026-06-01" <= a[1]["signed"] <= "2026-06-15" for a in cloud.values()),
      "the four commercial cloud agreements: signed in the last two weeks of May 2026, funded by the middle of June")
g13 = actions("HQ08832590013")
check(g13[-1]["obligated"] > total("HQ08832590012") + total("HQ08832590014") + total("HQ08832590015"), "Google's $20 million action alone exceeds the other three labs' totals combined")
check(len(actions("HQ08832590012")) == 6 and sum(1 for r in actions("HQ08832590012")[1:] if r["obligated"] > 0) == 1, "OpenAI: modified five times, funded once more")
check(len(g13) == 6 and all(r["obligated"] > 0 for r in g13[1:]), "Google: five later obligations, all positive (one of them $0.47)")

# other federal money to the same vendors that IS in FPDS
check(total("HB00012690001") == 3500000.00 and actions("HB00012690001")[0]["vendor"].startswith("OPENAI") and "CYBER COMMAND" in actions("HB00012690001")[0]["agency"].upper(),
      "OpenAI: US Cyber Command OT $3,500,000 (AI PLATFORM TRAINING)")
check(total("HR0011269E021") == 500000.00 and "DARPA" in actions("HR0011269E021")[0]["agency"].upper(), "OpenAI: DARPA other-transaction IDV $500,000 obligated")
st = actions("19PCRD26K4661")
check(st and st[0]["vendor"] == "ANTHROPIC, PBC" and st[0]["obligated"] == 18960.00 and st[0]["type"] == "PURCHASE ORDER" and st[0]["desc"] == "CLAUDE AI" and st[0]["signed"] == "2026-02-11",
      "Anthropic: the State Department purchase order, $18,960, CLAUDE AI, 2026-02-11, in FPDS too")

# ── 3. USAspending: the absence, as bravo's API pulls of 2026-09-21 (c6623) recorded it ─────────────────────────
print("\n== USAspending ==")
scan = json.load(open(SRC / "usaspending_vendor_scan.json", encoding="utf-8"))
def results_of(v):
    # the vendor scan saved each group's result LIST directly; the award-id lookup saved the whole response dict
    if isinstance(v, list):
        return v
    if isinstance(v, dict):
        return v.get("results") or []
    return []
for vendor in ("OPENAI", "X.AI", "XAI CORP"):
    keys = [k for k in scan if k.startswith(vendor + "/")]
    empties = [k for k in keys if not results_of(scan[k])]
    check(len(keys) == 3 and len(empties) == 3, f"USAspending recipient search {vendor!r}: 0 results in each of the contracts, IDV and other groups ({len(keys)} groups)")
ac = results_of(scan["ANTHROPIC/contracts"]); ai = results_of(scan["ANTHROPIC/idv"]); ao = results_of(scan["ANTHROPIC/other"])
check(len(ac) == 1 and abs(float(ac[0]["Award Amount"]) - 18960.00) < 0.01 and ac[0]["Award ID"] == "19PCRD26K4661" and ac[0]["Description"] == "CLAUDE AI"
      and ac[0]["Awarding Agency"] == "Department of State" and ac[0]["Start Date"] == "2026-02-11",
      f"USAspending recipient search 'ANTHROPIC', contracts group: one result, $18,960.00, 19PCRD26K4661, CLAUDE AI, State, 2026-02-11 (found {len(ac)})")
check(len(ai) == 1 and float(ai[0]["Award Amount"]) == 0.0 and "PHILANTHROPIC" in ai[0]["Recipient Name"].upper() and not ao,
      "USAspending recipient search 'ANTHROPIC', IDV group: one $0 GSA schedule for a consulting firm whose name contains the string (a substring match), and nothing in the other group")
look = json.load(open(SRC / "usaspending_piid_lookup.json", encoding="utf-8"))
for piid in ("HQ08832590012", "HQ08832590013", "HQ08832590014", "HQ08832590015", "HQ0883269E001"):
    ks = [k for k in look if k.startswith(piid + "/")]
    check(len(ks) == 2 and all(not results_of(look[k]) for k in ks), f"USAspending award-id lookup {piid}: 0 results in the contract and IDV groups")
pres = results_of(look["W9128Z26FA001/contracts"])
check(len(pres) == 1 and abs(float(pres[0]["Award Amount"]) - 545578476.94) < 0.01 and pres[0]["Recipient Name"] == "PALANTIR USG INC"
      and pres[0]["Awarding Sub Agency"] == "Department of the Army" and pres[0]["Start Date"] == "2025-10-15",
      "USAspending award-id lookup W9128Z26FA001: Palantir USG Inc, $545,578,476.94, Department of the Army, 2025-10-15")
e422 = look["award_type_groups_422"]
body = e422["body"] if isinstance(e422, dict) else str(e422)
check(isinstance(e422, dict) and e422.get("_err") == 422 and "'award_type_codes' must only contain types from one group." in body,
      "the API answered a mixed-group request with 422: \"'award_type_codes' must only contain types from one group.\"")
check('"award_type_groups":{"contracts":{"A":"BPA Call","B":"Purchase Order","C":"Delivery Order","D":"Definitive Contract"},"loans":' in body,
      "the 422 body lists its award-type groups, beginning contracts A/B/C/D (BPA call, purchase order, delivery order, definitive contract) and loans")
check("other transaction" not in body.lower() and "OTHER_TRANSACTION" not in body and "\"OT" not in body,
      "no group in the saved 422 body is an other-transaction group (the saved capture ends inside the IDV group; the groups USAspending documents are contracts, IDVs, grants, loans, direct payments and other assistance)")

# ── 4. the quotations, verbatim in the saved primaries ──────────────────────────────────────────────────────────
print("\n== quotations ==")
check("availability of DoD data entered into FPDS-NG is subject to a 90-day delay" in html_text("fpds_dodDataAvailability.html"), "FPDS: 'availability of DoD data entered into FPDS-NG is subject to a 90-day delay'")
atd = pdf_pages("usaspending_about_the_data.pdf")
check(len(atd) >= 9 and "Publication of Department of Defense (DOD) and U.S. Army Corps of Engineers (USACE) data in FPDS is delayed 90 days" in atd[8],
      "USAspending About the Data, p. 9: 'Publication of Department of Defense (DOD) and U.S. Army Corps of Engineers (USACE) data in FPDS is delayed 90 days'")
check("a web-based tool for federal agencies to report contract and OTA actions" in html_text("gao_25_107546.html"), "GAO-25-107546: FPDS is 'a web-based tool for federal agencies to report contract and OTA actions'")
xai = pdf_pages("fedscoop_xai_gsa_agreement.pdf")
check("AIQ Phase LLC dba xAI organized under the laws of the State of Wyoming" in xai[0], "xAI GSA agreement, p. 1: 'AIQ Phase LLC dba xAI organized under the laws of the State of Wyoming'")
ez = html_text("envzone_aiq.html")
check("AIQ Phase LLC, San Francisco, California, has been awarded a fixed amount, prototype, other transaction agreement (HQ0883-25-9-0015) with a value of $200,000,000" in ez, "DoD 14 July 2025 announcement sentence (as carried by the award-tracking page)")
check("Fiscal 2025 research, development, test and evaluation funds in the amount of $1,999,998 are being obligated at time of award" in ez, "DoD announcement: '$1,999,998 are being obligated at time of award'")
# packaging (scout bv2, 2026-09-22): the department's own release, reached by a headed browser at war.gov (article 4243673,
# the July 14, 2025 contracts page; defense.gov now redirects there) and saved beside the mirror
dod = html_text("dod_contracts_20250714_wargov.html")
check("AIQ Phase LLC, San Francisco, California, has been awarded a fixed amount, prototype, other transaction agreement (HQ0883-25-9-0015) with a value of $200,000,000" in dod, "DoD 14 July 2025 release, the department's own page (war.gov article 4243673): the AIQ Phase sentence")
check("Fiscal 2025 research, development, test and evaluation funds in the amount of $1,999,998 are being obligated at time of award" in dod, "DoD release, the department's own page: '$1,999,998 are being obligated at time of award'")
check("This contract, with a $200 million ceiling" in html_text("openai_for_government.html"), "OpenAI, Introducing OpenAI for Government: 'This contract, with a $200 million ceiling'")
check("each with a $200M ceiling" in html_text("breakingdefense_0714.html"), "Breaking Defense quoting the CDAO release: 'each with a $200M ceiling'")
check("$200 million prototype other transaction agreement to OpenAI Public Sector" in html_text("defensescoop_openai_0617.html"), "DefenseScoop 17 June 2025: '$200 million prototype other transaction agreement to OpenAI Public Sector'")
it = html_text("intercept_20260908.html")
check("each worth up to $200 million" in it and "later amendments" in it and "improve military advantage, military utility, or enhance military decision making" in it, "The Intercept, 8 September 2026: the three quoted phrases")

print("\nRESULT:", ("ALL CHECKS PASS (%d)" % 0) if not FAILS else "%d FAIL(S)" % len(FAILS))
for f in FAILS:
    print("  -", f)
sys.exit(1 if FAILS else 0)