#!/usr/bin/env python3
"""Reproduce exploratory phase3 from monthly-panel.csv. Requires numpy and scipy.
Run in repository: python scripts/build-20mph-phase3.py
Standalone: python reproduce.py --input-dir DIR --output-dir DIR
All selection/weights use pre-policy collisions; outcomes are already public.
"""
import argparse,csv,hashlib,io,json,pathlib
import numpy as np
from scipy.optimize import minimize
from scipy.stats import norm
ROOT=pathlib.Path(__file__).resolve().parents[1]
PINS={'monthly-panel.csv': '2503110cc962917778bdd93ab41bc860fdb89412c6e9bc6653187d4ae0bf4ce5', 'annual-controls.json': 'bb15c481f8810194541931daa761918c1949289e61a0ecc421ac482e136e2934', 'analysis-specification.json': 'd2e8657eaf4bdac49354180238113314e77c8bd365f987d1c025b956a8d0eb79'}
def validate_inputs(folder):
 for name,expected in PINS.items():
  if digest(folder/name)!=expected:raise ValueError("Input hash mismatch: "+name)
def digest(p):return hashlib.sha256(p.read_bytes()).hexdigest()
def prepare(folder):
 rows=list(csv.DictReader(io.StringIO((folder/'monthly-panel.csv').read_text())));months=sorted({r['month'] for r in rows});regions=sorted({r['region'] for r in rows if r['region']!='Wales'})
 arrays={}
 for scope in ('all','20_30'):
  for outcome in ('collisions','casualties'):
   d={(r['month'],r['region']):float(r[outcome]) for r in rows if r['scope']==scope}
   arrays[(scope,outcome)]=np.array([[d[m,r] for r in ['Wales']+regions] for m in months])
 return months,regions,arrays
def weights(a,pre,columns):
 # Normalize to 2015 mean, a constant independent of post outcomes.
 normed=a/a[:12].mean(axis=0);x=normed[pre][:,columns];y=normed[pre,0];k=len(columns)
 obj=lambda w:np.mean((x@w-y)**2)+.01*np.sum((w-1/k)**2)
 jac=lambda w:2*x.T@(x@w-y)/len(y)+.02*(w-1/k)
 r=minimize(obj,np.full(k,1/k),jac=jac,method='SLSQP',bounds=[(0,1)]*k,constraints={'type':'eq','fun':lambda w:w.sum()-1,'jac':lambda w:np.ones(k)},options={'ftol':1e-12,'maxiter':1000})
 assert r.success,r.message
 return r.x
def hac(x,e,t,lag=12):
 z=x*e[:,None];s=z.T@z
 for l in range(1,lag+1):
  ix={v:i for i,v in enumerate(t)};pairs=[(i,ix[v-l]) for i,v in enumerate(t) if v-l in ix]
  if pairs:
   p,q=np.array(pairs).T;cross=z[p].T@z[q];s+=(1-l/(lag+1))*(cross+cross.T)
 bread=np.linalg.pinv(x.T@x)
 return bread@s@bread*len(t)/(len(t)-x.shape[1])
def block_errors(e,t,draws,seed,block=12):
 rng=np.random.default_rng(seed);runs=np.split(np.arange(len(t)),np.where(np.diff(t)>1)[0]+1);out=np.empty((draws,len(t)))
 for run in runs:
  # Stationary bootstrap: random block lengths avoid a full-run rotation
  # fixing the sample mean when the run happens to have12 observations.
  z=e[run]-e[run].mean();n=len(run);ix=rng.integers(0,n,size=draws)
  for pos in run:
   out[:,pos]=z[ix];restart=rng.random(draws)<1/block
   ix=np.where(restart,rng.integers(0,n,size=draws),(ix+1)%n)
 return out
def model(months,a,columns,w,pre,post,trend=True,slope=False,draws=1999,seed=20261104,pooled=False):
 t0=np.arange(len(months));sel=np.where(pre|post)[0];t=t0[sel];p=post[sel].astype(float)
 normed=a/a[:12].mean(axis=0);c=normed[:,columns]@w
 if pooled:c=a[:,columns].sum(axis=1)/a[:12,columns].sum(axis=1).mean()
 y=np.log(normed[sel,0]/c[sel]);x=np.column_stack([np.ones(len(t))]+[(t%12==m).astype(float) for m in range(1,12)]+([t/12] if trend else [])+[p]+([p*(t-t[post[sel]][0])/12] if slope else []));j=12+int(trend)
 assert np.linalg.matrix_rank(x)==x.shape[1]
 pinv=np.linalg.pinv(x);b=pinv@y;e=y-x@b;cov=hac(x,e,t);se=np.sqrt(max(cov[j,j],0));boot=b[:,None]+pinv@block_errors(e,t,draws,seed).T
 xp=x[post[sel]];x0=xp.copy();x0[:,j:]=0;scale=c[sel][post[sel]]*a[:12,0].mean();smear=float(np.exp(e[pre[sel]]).mean())
 cf=scale*np.exp(x0@b)*smear;fitted=scale*np.exp(xp@b)*smear
 bcf=scale[:,None]*np.exp(x0@boot)*smear;bf=scale[:,None]*np.exp(xp@boot)*smear
 pct=(fitted.sum()/cf.sum()-1)*100;bpct=(bf.sum(axis=0)/bcf.sum(axis=0)-1)*100;absolute=float((fitted-cf).sum());ba=(bf-bcf).sum(axis=0)
 # Main level model's HAC interval. Slope variant reports bootstrap average contrast only.
 hci=[float(100*np.expm1(b[j]-1.96*se)),float(100*np.expm1(b[j]+1.96*se))] if not slope else None
 return {'pct':float(pct),'ci':[float(v) for v in np.quantile(bpct,[.025,.975])],'hacCI':hci,'absolute':absolute,'absoluteCI':[float(v) for v in np.quantile(ba,[.025,.975])],'observedPost':int(a[post,0].sum()),'fittedPost':float(fitted.sum()),'noBreakFittedPost':float(cf.sum()),'preMonths':int(pre.sum()),'postMonths':int(post.sum()),'bootstrapDraws':draws,'blockMonths':12,'conditionalMdePct':None if slope else float(100*(1-np.exp(-(1.96+.841621)*np.std(boot[j],ddof=1)))),'coefficient':float(b[j]),'residualSD':float(e.std(ddof=x.shape[1])),'weights':w.tolist(),'comparison':c.tolist(),'monthlyFitted':[{'month':months[i],'observed':int(a[i,0]),'fitted':float(v),'noBreakFitted':float(n)} for i,v,n in zip(np.where(post)[0],fitted,cf)]}
def clean(r):return {k:v for k,v in r.items() if k not in ('weights','comparison','monthlyFitted')}
def run(inp,out):
 validate_inputs(inp)
 out.mkdir(parents=True,exist_ok=True);months,regions,arrays=prepare(inp);n=len(months);tt=np.arange(n);years=np.array([int(m[:4]) for m in months]);pre=(years<=2019)|((years>=2022)&(np.array(months)<'2023-09'));post=np.array(months)>='2023-10';columns=list(range(1,len(regions)+1));ac=arrays[('all','collisions')]
 assert pre.sum()==80 and post.sum()==27 and n==132
 w=weights(ac,pre,columns);training=years<=2019;holdout=(years>=2022)&(np.array(months)<'2023-09');wh=weights(ac,training,columns);normed=ac/ac[:12].mean(axis=0);fit=normed[:,columns]@w;hf=normed[:,columns]@wh
 balance=[{'region':r,'weight':float(w[i]),'holdoutTrainingWeight':float(wh[i]),'baseline2015MonthlyCollisions':float(ac[:12,i+1].mean()),'preIndexRmse':float(np.sqrt(np.mean((normed[pre,i+1]-normed[pre,0])**2))),'policyStatus':'Not verified untreated'} for i,r in enumerate(regions)]
 results={};sensitivity=[];placebos=[];diagnostics={};quarterly=[]
 for outcome in ('collisions','casualties'):
  a=arrays[('all',outcome)];r=model(months,a,columns,w,pre,post);results[outcome]=r
  idx=a/a[:12].mean(axis=0);c=np.array(r['comparison']);yp=np.log(idx[pre,0]/c[pre]);tp=tt[pre];xp=np.column_stack([np.ones(len(tp))]+[(tp%12==m).astype(float) for m in range(1,12)]+[tp/12]);b=np.linalg.lstsq(xp,yp,rcond=None)[0];se=float(np.sqrt(hac(xp,yp-xp@b,tp)[-1,-1]));z=float(b[-1]/se)
  diagnostics[outcome]={'preRelativeAnnualPct':float(100*np.expm1(b[-1])),'preRelativeAnnualCI':[float(100*np.expm1(b[-1]-1.96*se)),float(100*np.expm1(b[-1]+1.96*se))],'preTrendP':float(2*norm.sf(abs(z))),'warning':'A small pretrend p-value flags nonparallel trends. A large value does not establish parallel trends.'}
  for q in range(44):
   s=slice(q*3,q*3+3);quarterly.append({'outcome':outcome,'period':f'{2015+q//4} Q{q%4+1}','year':2015+q//4,'quarter':q%4+1,'walesIndex':float(100*idx[s,0].mean()),'comparisonIndex':float(100*c[s].mean()),'walesCount':int(a[s,0].sum()),'relativeIndex':float(100*np.exp(np.log(idx[s,0]/c[s]).mean()))})
  scenarios=[('main','Main: seasonal + relative trend',columns,pre,post,True,False,False,'all'),('no-trend','No relative trend',columns,pre,post,False,False,False,'all'),('post-slope','Separate post-policy slope',columns,pre,post,True,True,False,'all'),('pandemic','Include pandemic years',columns,np.array(months)<'2023-09',post,True,False,False,'all'),('recent-pre','Pre-period2022–Aug2023 only',columns,holdout,post,True,False,False,'all'),('end-2024','End follow-up December2024',columns,pre,post&(years<=2024),True,False,False,'all'),('no-london','Omit London', [i+1 for i,r in enumerate(regions) if r!='London'],pre,post,True,False,False,'all'),('omit-rollout-regions','Omit London, South East, South West',[i+1 for i,r in enumerate(regions) if r not in ('London','South East','South West')],pre,post,True,False,False,'all'),('pooled-england','Pooled England counts',columns,pre,post,True,False,True,'all'),('speed-proxy','Recorded20/30mph category proxy',columns,pre,post,True,False,False,'20_30')]+[(f'leave-{i}',f'Leave out {region}',[j for j in columns if j!=i+1],pre,post,True,False,False,'all') for i,region in enumerate(regions)]
  for sid,label,cc,pr,po,tr,sl,pool,scope in scenarios:
   # Baseline collision weights are retained for the speed-category and casualty checks.
   ww=w if cc==columns and np.array_equal(pr,pre) else weights(ac,pr,cc)
   rr=r if sid=='main' else model(months,arrays[(scope,outcome)],cc,ww,pr,po,trend=tr,slope=sl,pooled=pool,draws=999,seed=20261104)
   sensitivity.append({'id':sid,'label':label,'outcome':outcome,'scope':scope,**clean(rr)})
  controls=json.loads((inp/'annual-controls.json').read_text())['records'];scaled={k:v.copy() for k,v in arrays.items()}
  for measure in ('collisions','casualties'):
   for control in controls:
    if control['measure']==measure and control['area'] in regions:
     year=years==control['year'];j=regions.index(control['area'])+1
     scaled[('all',measure)][year,j]*=control['value']/scaled[('all',measure)][year,j].sum()
  ww=weights(scaled[('all','collisions')],pre,columns);rr=model(months,scaled[('all',outcome)],columns,ww,pre,post,draws=999)
  sensitivity.append({'id':'annual-geography-controls','label':'Scale English region-years to official annual totals','outcome':outcome,'scope':'all',**clean(rr)})
  for date in ('2017-01','2017-10','2018-10'):
   start=months.index(date);pr=tt<start;po=(tt>=start)&(tt<start+12);ww=weights(ac,pr,columns);rr=model(months,a,columns,ww,pr,po,draws=999,seed=20261104)
   placebos.append({'date':date,'outcome':outcome,**clean(rr),'note':'Fake intervention;12 months follow-up wholly before Welsh national policy. Weights trained only before fake date.'})
  rr=model(months,a,columns,w,pre,post&(tt<months.index('2023-10')+12),draws=999,seed=20261104)
  placebos.append({'date':'2023-10','outcome':outcome,**clean(rr),'note':'Actual policy period, first12 full months; comparison for the12-month placebo horizon.'})
 output={'version':'20mph-phase3-v3','analysisDate':'2026-11-04','causalGate':'Not met','confidence':'Certain for reconciled counts; exploratory and model-dependent for controlled associations; policy attribution not established.','source':'DfT complete collision file, fetched4 November2026; final through2025. StatsWales and provisional2026 are not spliced into the model.','preMonths':80,'postMonths':27,'regions':regions,'balance':balance,'fit':{'preIndexRmse':float(np.sqrt(np.mean((fit[pre]-normed[pre,0])**2))),'training2015to2019Rmse':float(np.sqrt(np.mean((hf[training]-normed[training,0])**2))),'holdout2022toAug2023Rmse':float(np.sqrt(np.mean((hf[holdout]-normed[holdout,0])**2))),'holdoutMeanGapPctPoints':float(100*np.mean(normed[holdout,0]-hf[holdout]))},'diagnostics':diagnostics,'main':{k:clean(v) for k,v in results.items()},'quarterly':quarterly,'sensitivity':sensitivity,'placebos':placebos,'monthlyFitted':{k:v['monthlyFitted'] for k,v in results.items()},'inputSha256':digest(inp/'monthly-panel.csv'),'specificationSha256':digest(inp/'analysis-specification.json')}
 (out/'results.json').write_text(json.dumps(output,indent=2,allow_nan=False)+'\n')
 for name,rows in [('sensitivity.csv',sensitivity),('placebos.csv',placebos),('quarterly-index.csv',quarterly),('comparison-balance.csv',balance)]:
  with (out/name).open('w',newline='') as f:
   keys=list(rows[0]);wr=csv.DictWriter(f,keys);wr.writeheader();wr.writerows(rows)
 print(json.dumps({'main':output['main'],'fit':output['fit'],'diagnostics':diagnostics},indent=2))
if __name__=='__main__':
 p=argparse.ArgumentParser();p.add_argument('--input-dir',type=pathlib.Path,default=ROOT/'public/research/20mph/v3');p.add_argument('--output-dir',type=pathlib.Path,default=ROOT/'public/research/20mph/v3');args=p.parse_args();run(args.input_dir,args.output_dir)
