# -*- coding: utf-8 -*- """Census: final rules whose ACTION is only to move a date, Federal Register, 2017-2026. Reproduction script for the Federal Register date-move census. Run: python fr_date_moves.py Method, stated so it can be argued with: - Population: documents of type RULE (final rules) published 1 January to 15 September of each year, so every year is the same window. Counts come from the Federal Register API's own `count`, not from anything this script infers. - Classifier: each document's own `action` field, which is the agency's one-line statement of what the document does ("Final rule; delay of effective date."). A document counts as a DATE MOVE when that field matches DATE_MOVE_RE and does not match CARVE_OUT_RE. - The classifier reads the ACTION field, never the title. Titles describe the subject; the action field describes the operation, and it is the operation being counted. Known limits, which belong in the essay and not in a footnote: - A rule that moves a date inside a larger substantive action is NOT counted. This is a floor, not a total. - `action` is free text. Agencies word it inconsistently, and a date move phrased in a way DATE_MOVE_RE does not anticipate is missed. UNMATCHED_SAMPLE prints action strings that contain "date" but did not match, so the misses can be inspected rather than assumed away. - 2026 is a partial year by construction (through 15 September). So is every comparison year. """ import json import re import sys import time import urllib.parse import urllib.request API = "https://www.federalregister.gov/api/v1/documents.json" UA = {"User-Agent": "research-census/1.0", "Accept-Encoding": "gzip, deflate"} # The operation: postponing, delaying, extending or staying a date that already exists. DATE_MOVE_RE = re.compile( r"(delay(ing)?|postpon\w+|extension of|extend\w*|stay(ing)? of|suspension of)\b[^.;]{0,60}" r"\b(effective|compliance|applicability|implementation|submission|reporting)\b" r"|" r"\b(effective|compliance|applicability|implementation|submission|reporting)\b[^.;]{0,40}" r"\b(date|period|deadline)s?\b[^.;]{0,40}\b(delay\w*|postpon\w+|extend\w*|extension)\b", re.I) # A document that also does substantive work is not "only" a date move. CARVE_OUT_RE = re.compile(r"\b(amendment|revision|correction|interim final rule;? *(and )?request)\b", re.I) def fetch(params, tries=4): url = API + "?" + urllib.parse.urlencode(params, doseq=True) for attempt in range(tries): try: req = urllib.request.Request(url, headers=UA) with urllib.request.urlopen(req, timeout=90) as r: raw = r.read() if r.headers.get("Content-Encoding") == "gzip": import gzip raw = gzip.decompress(raw) return json.loads(raw.decode("utf-8")) except Exception as exc: # noqa: BLE001 - a transient API failure must not if attempt == tries - 1: # silently become a zero in a published count raise print(" retry %d after %s" % (attempt + 1, exc), file=sys.stderr) time.sleep(3 * (attempt + 1)) return {} def year_window(year): return "%d-01-01" % year, "%d-09-15" % year def census(year): lo, hi = year_window(year) base = { "conditions[type][]": "RULE", "conditions[publication_date][gte]": lo, "conditions[publication_date][lte]": hi, "fields[]": ["action", "title", "document_number", "publication_date", "agencies"], "per_page": 1000, "order": "oldest", } total, moves, unmatched_with_date, by_agency, examples = None, [], [], {}, [] page = 1 while True: d = fetch(dict(base, page=page)) if total is None: total = d.get("count", 0) results = d.get("results") or [] if not results: break for doc in results: action = (doc.get("action") or "").strip() if not action: continue if DATE_MOVE_RE.search(action) and not CARVE_OUT_RE.search(action): moves.append(doc) # Each rule counted ONCE, under its first-listed agency. A rule is often filed under a # department and its sub-agency (Interior and Surface Mining, Labor and OSHA), and # counting every listing would make the agency totals add up to more rules than exist. ags = doc.get("agencies") or [] name = (ags[0].get("name") or ags[0].get("raw_name") or "?") if ags else "?" by_agency[name] = by_agency.get(name, 0) + 1 if len(examples) < 6: examples.append((doc["document_number"], doc["publication_date"], action[:90], (doc.get("title") or "")[:90])) elif re.search(r"\bdate\b", action, re.I): unmatched_with_date.append(action[:110]) if len(results) < base["per_page"]: break page += 1 if page > 12: # API caps deep paging; refuse to report a raise SystemExit("PAGE CAP HIT for %d - counts would be partial" % year) time.sleep(1) return {"year": year, "window": (lo, hi), "total_rules": total, "moves": moves, "by_agency": by_agency, "examples": examples, "unmatched_with_date": unmatched_with_date} def main(): years = list(range(2017, 2027)) rows = [] for y in years: c = census(y) n, t = len(c["moves"]), c["total_rules"] rows.append((y, n, t, 100.0 * n / t if t else 0.0)) print("%d %s..%s date-move rules %4d of %5d final rules %5.2f%%" % (y, c["window"][0], c["window"][1], n, t, 100.0 * n / t if t else 0.0)) if y == 2026: print("\n AGENCIES 2026 (each rule once, under its first-listed agency):") for name, k in sorted(c["by_agency"].items(), key=lambda kv: -kv[1]): print(" %4d %s" % (k, name)) print(" %4d total, which equals the date-move rules for 2026" % sum(c["by_agency"].values())) print("\n EXAMPLES 2026 (document, published, action, title):") for e in c["examples"]: print(" %s %s | %s | %s" % e) print("\n UNMATCHED_SAMPLE (action mentions a date, classifier said no) %d total:" % len(c["unmatched_with_date"])) for s in c["unmatched_with_date"][:8]: print(" - %s" % s) time.sleep(1) print("\nYEAR DATE-MOVE TOTAL SHARE") for y, n, t, p in rows: print("%4d %9d %5d %5.2f%%" % (y, n, t, p)) if __name__ == "__main__": main()