"""Scores v5 pre-registered forecast of the November 3, 2026 Governor's race by town (plan decision I).

    SCORES_OUT=<scores_v4 folder> SCORES_POLLS=<site-design/polling-page/polling_data.json> ASOF=2026-10-30 python3 v4_forecast.py

Forecast margin (D minus R, two-party) for each rated town = the town's relative lean (scores_towns.csv; v4 called it Natural Lean, v5 adds the district lean back, see v5_scores.py) + the statewide two-party
polling margin of the race on the as-of date (the site's polling-average rule: each pollster's last poll in the final 21 days before
the as-of date, weighted by grade x sqrt(n/600) x sponsor; polling-page/README.md). Also written: the "last presidential margin"
rule (the town's 2024 President margin as the forecast) that the ratings will be compared with, and the top-50 most competitive towns
under each rule (the targeting back-test).

Scoring after certification (both against the official town returns, two-party margin, towns with a result line):
  geographic   replace the polling margin with the actual statewide two-party margin: does the relative lean place towns correctly?
               (mean absolute error, r, and the same for the 2024-President rule shifted by the same statewide swing)
  operational  the file as frozen: mean absolute error and the share of towns whose winner was called;
  targeting    of the 50 towns each rule called most competitive, how many were decided by under 5 points.
The frozen file is forecast_<asof>.csv in forecast_2026-11-03/; nothing in it changes after the freeze date.
"""
import os, json, math, pandas as pd
OUT = os.environ.get('SCORES_OUT', '/home/claude/v5/out')
POLLS = os.environ.get('SCORES_POLLS', '/mnt/user-data/uploads/Maine Political Data Project/site-design/polling-page/polling_data.json')
ASOF = os.environ.get('ASOF', '2026-10-17')
RACE = os.environ.get('RACE', '2026|GOV|general||')
GRADE = {'A+': 1, 'A': 1, 'A-': .95, 'A/B': .9, 'B+': .85, 'B': .8, 'B-': .75, 'B/C': .7, 'C+': .65, 'C': .6, 'C-': .55, 'C/D': .5, 'D+': .45, 'D': .4, 'D-': .35, 'F': .2}
J = json.load(open(POLLS)); race = next(r for r in J['races'] if r['key'] == RACE)
dem = next(c['n'] for c in race['cands'] if c['p'] == 'DEM'); rep = next(c['n'] for c in race['cands'] if c['p'] == 'REP')
lo = (pd.Timestamp(ASOF) - pd.Timedelta(days=21)).strftime('%Y-%m-%d')
polls = [p for p in race['polls'] if lo <= p['e'] <= ASOF and dem in p['v'] and rep in p['v']]
last = {}
for p in sorted(polls, key=lambda p: p['e']): last[p['who']] = p
rows = []
for p in last.values():
    q = GRADE.get(p.get('g') or '', None)
    if q is None: q = 0.4 + 0.2 * min(3, int(p['r'])) if str(p.get('r') or '').isdigit() else 0.6
    n = float(p['n']) if str(p.get('n') or '').replace('.', '').isdigit() else 600.0
    w = q * math.sqrt(min(n, 3000) / 600) * (0.5 if any(k in (p.get('type') or '') for k in ('internal', 'Party', 'Partisan', 'advocacy')) else 1)
    d, r = p['v'][dem], p['v'][rep]
    rows.append(dict(pollster=p['who'], end=p['e'], n=int(n), type=p.get('type'), dem=d, rep=r, two_party_margin=100 * (d - r) / (d + r), weight=w))
P = pd.DataFrame(rows)
poll_margin = (P.two_party_margin * P.weight).sum() / P.weight.sum()
T = pd.read_csv(f'{OUT}/scores_towns.csv')
F = T[['unit_id', 'name', 'county', 'relative_lean', 'partisan_eai', 'margin_president_2024', 'two_party_votes_2024', 'size_class']].copy()
F['poll_margin_statewide'] = round(poll_margin, 1)
F['forecast_margin'] = (F.relative_lean + poll_margin).round(1)
F['forecast_winner'] = F.forecast_margin.map(lambda m: 'DEM' if m > 0 else 'REP')
F['rule_last_president_margin'] = F.margin_president_2024
F['competitive_rank_forecast'] = F.forecast_margin.abs().rank(method='first').astype('Int64')
F['competitive_rank_last_president'] = F.margin_president_2024.abs().rank(method='first').astype('Int64')
F['top50_forecast'] = F.competitive_rank_forecast <= 50; F['top50_last_president'] = (F.competitive_rank_last_president <= 50).fillna(False)
F['race'] = f'Governor 2026: {dem} (D) vs {rep} (R)'; F['asof'] = ASOF
d = f'{OUT}/forecast_2026-11-03'; os.makedirs(d, exist_ok=True)
F.to_csv(f'{d}/forecast_{ASOF}.csv', index=False); P.to_csv(f'{d}/polls_used_{ASOF}.csv', index=False)
open(f'{d}/README.md', 'w').write(__doc__ + f'\n\nAs of {ASOF}: {len(P)} pollsters\' last polls since {lo}; weighted statewide two-party margin {dem} minus {rep} = {poll_margin:+.1f}. '
                                  f'Towns forecast Democratic: {int((F.forecast_margin > 0).sum())} of {len(F)}; within 5 points: {int((F.forecast_margin.abs() < 5).sum())}. '
                                  f'Top-50 overlap between the two targeting rules: {int((F.top50_forecast & F.top50_last_president).sum())} towns.\n')
print(open(f'{d}/README.md').read()[-500:])