#!/usr/bin/env python3
"""Every figure in "How Long Was It Dark?" - one GET request, no API key, standard library only.
Usage: python hibp_dark_interval_charlie_cp20260910.py [saved_breaches.json]
Dark interval = AddedDate (the day Have I Been Pwned could answer) minus BreachDate (the day it happened)."""
import sys, json, statistics as st, collections, datetime as dt, urllib.request
URL = "https://haveibeenpwned.com/api/v3/breaches"
LAUNCH = dt.date(2013, 12, 4)  # the day the site went live; breaches before it were back-filled
if len(sys.argv) > 1:
    data = json.load(open(sys.argv[1], encoding="utf-8"))
else:
    data = json.load(urllib.request.urlopen(urllib.request.Request(URL, headers={"User-Agent": "dark-interval-study"}), timeout=60))
D = lambda s: dt.date.fromisoformat(s[:10])
rows = [dict(b, bd=D(b["BreachDate"]), ad=D(b["AddedDate"])) for b in data]
for r in rows:
    r["gap"] = (r["ad"] - r["bd"]).days
med = lambda xs: st.median(xs) if xs else float("nan")
share = lambda xs, f: 100 * sum(1 for x in xs if f(x)) / len(xs)
def summary(label, rs):
    g = sorted(r["gap"] for r in rs)
    print(f"{label}: n={len(g)} median={med(g):.0f}d p90={g[round(.9 * (len(g) - 1))]}d max={max(g)}d "
          f"<=90d {share(g, lambda x: x <= 90):.1f}% >1y {share(g, lambda x: x > 365):.1f}% >2y {share(g, lambda x: x > 730):.1f}%")
print("run", dt.date.today(), "| breaches", len(rows), "| negative gaps", sum(r["gap"] < 0 for r in rows), "| zero-day", sum(r["gap"] == 0 for r in rows))
summary("ALL", rows)
summary("HIBP-ERA (breached on or after launch)", [r for r in rows if r["bd"] >= LAUNCH])
pre = [r for r in rows if r["bd"] < LAUNCH]
print(f"pre-launch: n={len(pre)} median raw={med([r['gap'] for r in pre]):.0f}d, measured from launch day={med([(r['ad'] - LAUNCH).days for r in pre]):.0f}d")
print("\nby breach year: n, median, share catalogued within 90d / 365d (recent rows are upper bounds: an uncatalogued breach is absent)")
for y in sorted({r["bd"].year for r in rows}):
    g = [r["gap"] for r in rows if r["bd"].year == y]
    if len(g) >= 10:
        print(f"  {y} n={len(g):3d} median={med(g):5.0f}d  <=90d {share(g, lambda x: x <= 90):3.0f}%  <=365d {share(g, lambda x: x <= 365):3.0f}%")
print("\nby what was taken (n>=25): n, median, all years | breached 2014-2021 only, the era control")
cls = collections.defaultdict(list)
for r in rows:
    for c in r["DataClasses"]:
        cls[c].append(r)
for c, rs in sorted(cls.items(), key=lambda kv: -med([r["gap"] for r in kv[1]])):
    if len(rs) >= 25:
        era = [r["gap"] for r in rs if 2014 <= r["bd"].year <= 2021]
        print(f"  {c:28s} n={len(rs):4d} median={med([r['gap'] for r in rs]):4.0f}d | 2014-21 n={len(era):3d} median={med(era):4.0f}d")
print("\nby size (PwnCount): all | breached 2016 or later")
for lo, hi, lab in [(0, 1e4, "<10k"), (1e4, 1e5, "10k-100k"), (1e5, 1e6, "100k-1M"), (1e6, 1e7, "1M-10M"), (1e7, 1e8, "10M-100M"), (1e8, 1e13, ">100M")]:
    a = [r["gap"] for r in rows if lo <= r["PwnCount"] < hi]
    b = [r["gap"] for r in rows if lo <= r["PwnCount"] < hi and r["bd"].year >= 2016]
    print(f"  {lab:9s} n={len(a):3d} median={med(a):4.0f}d | 2016+ n={len(b):3d} median={med(b):4.0f}d")
print("\nflags: n, median")
for f in ["IsVerified", "IsSensitive", "IsMalware", "IsStealerLog", "IsSpamList", "IsFabricated"]:
    t = [r["gap"] for r in rows if r[f]]
    print(f"  {f:13s} true n={len(t):4d} median={med(t):4.0f}d | false median={med([r['gap'] for r in rows if not r[f]]):4.0f}d")
print("\nlongest:")
for r in sorted(rows, key=lambda r: -r["gap"])[:8]:
    print(f"  {r['gap']:5d}d {r['gap'] / 365.25:4.1f}y {r['Title'][:32]:32s} {r['bd']} -> {r['ad']} accounts={r['PwnCount']:,}")
print(f"\nprecision: {sum(r['bd'].day == 1 for r in rows)} breaches dated the 1st of a month, {sum(r['bd'].month == 1 and r['bd'].day == 1 for r in rows)} dated 1 January; "
      f"loaded on launch day: {[r['Name'] for r in rows if r['ad'] == LAUNCH]}")