"""Scores v5 checks (run after v5_scores.py; writes to $SCORES_OUT/checks/):
  1. campaign thresholds against the 2022 and 2024 State House results on the current district lines;
  2. registration margin on active voters vs active + inactive (Nov 2024), by size class;
  3. Natural Lean stability on non-overlapping periods (2010-2016 vs 2018-2024, equal weights, towns with 1,000+ votes);
  4. the 15% rule: races within 5 points of the threshold;
  5. the EAI weighting grid search is copied from the audit (a/eai_grid.csv) when present.
"""
import os, numpy as np, pandas as pd
OUT = os.environ.get('SCORES_OUT', '/home/claude/v5/out'); CHK = OUT + '/checks'
S = pd.read_pickle(f'{OUT}/v5_state.pkl'); RU, Wg, races = S['RU'], S['Wg'], S['races']
T = S['T']
lines = []
def say(*a):
    s = ' '.join(str(x) for x in a); print(s); lines.append(s)

# 1. thresholds vs State House results ---------------------------------------------------------
sh = pd.read_pickle('state_house_2008_2024.pkl'); sh = sh[sh.election_date >= '2022-01-01']
sh = sh[sh.col_role == 'candidate']
sh.loc[sh.writein.astype(str).str.lower() == 'true', 'cand_party'] = 'WRITEIN'
res = []
for cid, d in sh.groupby('contest_id'):
    dv = d[d.cand_party == 'DEM'].votes.sum(); rv = d[d.cand_party == 'REP'].votes.sum(); tot = d.votes.sum()
    if dv > 0 and rv > 0:
        res.append(dict(contest_id=cid, year=int(cid[:4]), hd=int(float(d.district.iloc[0])), margin=100 * (dv - rv) / (dv + rv), nonmajor=1 - (dv + rv) / tot))
res = pd.DataFrame(res)
hd = T['state_house_districts'].set_index('house_district')
res['nl'] = res.hd.map(hd.partisan_eai); res['eai'] = res.hd.map(hd.raw_vote_margin); res['cls'] = res.hd.map(hd.campaign_class)   # nl = Partisan EAI (neutral-year), eai = raw vote
res['won_D'] = res.margin > 0
say('1. State House 2022 and 2024, contested D-vs-R races on the current lines:', len(res), 'races')
for cls, g in res.groupby('cls'):
    fav = 'D' if cls.endswith('_D') else ('R' if cls.endswith('_R') else None)
    if fav: say(f'   {cls:24s} n={len(g):3d}  favored side won {int((g.won_D == (fav == "D")).sum())} ({100 * (g.won_D == (fav == "D")).mean():.0f}%)  mean |margin| {g.margin.abs().mean():.1f}')
    else: say(f'   {cls:24s} n={len(g):3d}  Democrats won {int(g.won_D.sum())} ({100 * g.won_D.mean():.0f}%)  mean |margin| {g.margin.abs().mean():.1f}')
# alternative thresholds: how often does the favored side lose, by |Natural Lean| band
res['band'] = pd.cut(res.nl.abs(), [0, 3, 5, 7, 10, 15, 20, 100], right=False)
say('   by |Partisan EAI| band (share won by the side the EAI favors):')
for b, g in res.groupby('band', observed=True):
    fav_won = ((g.nl >= 0) == g.won_D)
    say(f'     {str(b):10s} n={len(g):3d}  favored won {100 * fav_won.mean():.0f}%   State House margin vs Partisan EAI: r {g.margin.corr(g.nl):.2f}, vs raw vote: r {g.margin.corr(g.eai):.2f}')
say(f'   all races: State House margin vs Partisan EAI r {res.margin.corr(res.nl):.3f}; vs raw vote r {res.margin.corr(res.eai):.3f}; mean |margin - raw vote| {(res.margin - res.eai).abs().mean():.1f}; mean |margin - Partisan EAI| {(res.margin - res.nl).abs().mean():.1f}')
res.to_csv(f'{CHK}/state_house_vs_thresholds.csv', index=False)

# 2. active vs all ---------------------------------------------------------------------------------
REG = os.environ.get('SCORES_REG', '/home/claude/audit/reg/out'); SNAP = '2024-11-05'
act = pd.read_csv(f'{REG}/registration_town_wide.csv', low_memory=False); act = act[act.snapshot_date == SNAP].set_index('unit_id')
ina = pd.read_csv(f'{REG}/inactive_registration_town_wide.csv', low_memory=False); ina = ina[ina.snapshot_date == SNAP].set_index('unit_id')
tw = T['towns'].set_index('unit_id')
a = pd.DataFrame({'ma': 100 * (act.DEM - act.REP) / (act.DEM + act.REP)})
D = act.DEM.add(ina.DEM.reindex(act.index).fillna(0)); R = act.REP.add(ina.REP.reindex(act.index).fillna(0))
a['mall'] = 100 * (D - R) / (D + R); a = a.join(tw[['size_class', 'name']], how='inner'); a['diff'] = a.ma - a.mall
say('2. Registration margin, active vs active + inactive (Nov 2024):')
for cls, g in a.groupby('size_class'):
    say(f'   {cls:18s} n={len(g):3d}  mean active - all {g["diff"].mean():+.1f}  sd {g["diff"].std():.1f}  r {g.ma.corr(g.mall):.3f}')
say('   largest differences:', ', '.join(f'{r["name"]} {r["diff"]:+.1f}' for _, r in a.reindex(a["diff"].abs().sort_values(ascending=False).index).head(6).iterrows()))
a.to_csv(f'{CHK}/registration_active_vs_all.csv')

# 3. non-overlapping stability -----------------------------------------------------------------------
exec(open('v5_scores.py').read().split('# ------------------------------------------------------------------ 2.')[0].split('# ------------------------------------------------------------------ 1.')[0])  # helpers only
def cyc_nl(df):
    s = df.groupby(['unit_id', 'year', 'office'])[['D', 'R', 'refDR', 'tp']].sum().reset_index()
    s['nl'] = 100 * (s.D - s.R - s.refDR) / s.tp; s['h'] = s.office == 'U.S. House'
    out = {}
    for (u, y), g in s.groupby(['unit_id', 'year']):
        hh, oo = g[g.h], g[~g.h]
        out[(u, y)] = (2 / 3 * oo.nl.mean() + 1 / 3 * hh.nl.mean()) if len(hh) and len(oo) else g.nl.mean()
    return pd.Series(out)
c = cyc_nl(RU).unstack()
big = tw[tw.two_party_votes_2024 >= 1000].index
A = c.loc[c.index.isin(big), [2010, 2012, 2014, 2016]].mean(axis=1); B = c.loc[c.index.isin(big), [2018, 2020, 2022, 2024]].mean(axis=1)
j = pd.DataFrame({'a': A, 'b': B}).dropna(); rk = j.rank()
say(f'3. Relative lean, 2010-2016 vs 2018-2024 (equal weights), towns with 1,000+ two-party votes in 2024: n={len(j)}, r {j.a.corr(j.b):.3f}, '
    f'rank r {j.a.corr(j.b, method="spearman"):.3f}, moved more than 50 ranks: {int(((rk.a - rk.b).abs() > 50).sum())}, mean |change| {(j.b - j.a).abs().mean():.1f} points')
j.assign(name=j.index.map(tw.name)).to_csv(f'{CHK}/relative_lean_nonoverlapping.csv')

# 4. 15% rule edges -------------------------------------------------------------------------------------
e = races[(races.nonmajor_share.between(0.10, 0.20))][['year', 'office', 'district', 'dem', 'rep', 'nonmajor_share', 'largest_other', 'qualifies']]
say('4. Races with non-major share between 10% and 20% (the 15% rule decides):'); say(e.to_string(index=False) if len(e) else '   none')
e.to_csv(f'{CHK}/rule15_edges.csv', index=False)

# 5. grid search copy
src = '/home/claude/audit/a/eai_grid.csv'
if os.path.exists(src):
    g = pd.read_csv(src); g.to_csv(f'{CHK}/eai_weighting_grid.csv', index=False); say('5. EAI weighting grid search copied:', len(g), 'schemes (audit script b5.py)')
open(f'{CHK}/CHECKS.txt', 'w').write('\n'.join(lines) + '\n')