"""content-c6623-000: pull every FPDS-NG action for the four CDAO frontier-AI vendors from the public ezsearch ATOM feed
(no login), page through the feed, save the raw XML per query under sources/, and tabulate per agreement: base award,
each modification, obligated amount per action, running obligation, base-and-all-options value, contracting office,
and the description. The point is the OTHER TRANSACTION AGREEMENT rows for HQ0883 (the Office of the Secretary of
Defense / CDAO PIID prefix), which USAspending's award-type groups do not carry.
Run (from the run dir): python fpds_frontier_ota_c6623.py > fpds_frontier_ota_c6623.out.txt
"""
import re
import time
import urllib.parse
import urllib.request
from pathlib import Path

SRC = Path(__file__).with_name("sources")
SRC.mkdir(exist_ok=True)
UA = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/128.0 Safari/537.36"}
QUERIES = {
    "openai": 'VENDOR_FULL_NAME:"OPENAI PUBLIC SECTOR"',
    "anthropic": 'VENDOR_FULL_NAME:"ANTHROPIC"',
    "xai": 'VENDOR_FULL_NAME:"X.AI"',
    "xai2": 'VENDOR_FULL_NAME:"XAI"',
    "google_ps": 'VENDOR_FULL_NAME:"GOOGLE PUBLIC SECTOR" CONTRACTING_AGENCY_ID:"9700"',
    "hq0883_2025": 'PIID:"HQ088325*"',
    "hq0883_2026": 'PIID:"HQ088326*"',
    "openai_free": "OPENAI",
    "anthropic_free": "ANTHROPIC",
    "xai_free": "XAI",
}


def fetch_all(q):
    out, start = [], 0
    while True:
        url = ("https://www.fpds.gov/ezsearch/FEEDS/ATOM?FEEDNAME=PUBLIC&q=" + urllib.parse.quote(q) + "&start=%d" % start)
        b = urllib.request.urlopen(urllib.request.Request(url, headers=UA), timeout=120).read().decode("utf-8", "replace")
        out.append(b)
        ents = re.findall(r"<entry>", b)
        if len(ents) < 10 or start > 400:
            break
        start += 10
        time.sleep(0.6)
    return out


def g(e, p):
    m = re.search(p, e, re.S)
    return m.group(1).strip() if m else ""


rows = {}
for key, q in QUERIES.items():
    try:
        pages = fetch_all(q)
    except Exception as ex:  # noqa: BLE001
        print("ERR", key, ex)
        continue
    (SRC / f"fpds_{key}.xml").write_text("\n<!-- PAGE BREAK -->\n".join(pages), encoding="utf-8")
    n = 0
    for b in pages:
        for e in re.findall(r"<entry>(.*?)</entry>", b, re.S):
            n += 1
            piid = g(e, r"<ns1:PIID>([^<]+)")
            ref = g(e, r"<ns1:referencedIDVID>\s*<ns1:PIID>([^<]+)")
            mod = g(e, r"<ns1:modNumber>([^<]+)") or "0"
            row = {
                "vendor": g(e, r"<ns1:vendorName>([^<]+)"), "piid": piid, "ref_idv": ref, "mod": mod,
                "signed": g(e, r"<ns1:signedDate>([^<]+)")[:10], "obligated": g(e, r"<ns1:obligatedAmount>([^<]+)"),
                "base_all_options": g(e, r"<ns1:baseAndAllOptionsValue>([^<]+)"),
                "base_exercised": g(e, r"<ns1:baseAndExercisedOptionsValue>([^<]+)"),
                "office": g(e, r'<ns1:contractingOfficeAgencyID[^>]*name="([^"]+)"'),
                "office_id": g(e, r'<ns1:contractingOfficeID[^>]*name="([^"]+)"'),
                "type": g(e, r'<ns1:contractActionType[^>]*description="([^"]+)"'),
                "reason_mod": g(e, r'<ns1:reasonForModification[^>]*description="([^"]+)"'),
                "desc": g(e, r"<ns1:descriptionOfContractRequirement>([^<]+)"),
                "completion": g(e, r"<ns1:currentCompletionDate>([^<]+)")[:10],
                "ultimate": g(e, r"<ns1:ultimateCompletionDate>([^<]+)")[:10],
                "naics": g(e, r"<ns1:principalNAICSCode[^>]*>([^<]+)"),
                "psc": g(e, r"<ns1:productOrServiceCode[^>]*>([^<]+)"),
            }
            rows[(piid, mod, row["signed"], row["vendor"])] = row
    print(f"== {key}: {q} -> {n} entries over {len(pages)} page(s)")

print("\n=== per agreement (deduplicated across queries) ===")
by = {}
for r in rows.values():
    by.setdefault((r["vendor"], r["piid"]), []).append(r)
for (vendor, piid), acts in sorted(by.items()):
    acts.sort(key=lambda r: (r["signed"], r["mod"]))
    tot = 0.0
    print(f"\n{vendor} | {piid} | {acts[0]['type']} | office {acts[0]['office']} ({acts[0]['office_id']}) | {acts[0]['desc'][:70]}")
    for r in acts:
        try:
            tot += float(r["obligated"] or 0)
        except ValueError:
            pass
        print(f"   {r['signed']} mod {r['mod']:>6} obligated {float(r['obligated'] or 0):>15,.2f} running {tot:>15,.2f} | base+all-options {r['base_all_options']:>15} | completion {r['completion']} ultimate {r['ultimate']} | {r['reason_mod'][:40]} | {r['desc'][:50]}")
    print(f"   TOTAL OBLIGATED on {piid}: {tot:,.2f}")