#!/usr/bin/env python3
"""Reproduce the 20mph descriptive pilot from pinned inputs (Python standard library).
Run: python build-20mph-phase2.py --download --input-dir inputs --output-dir reproduced
"""
import argparse,csv,gzip,hashlib,io,json,pathlib,statistics,urllib.request,urllib.error,importlib.util
SITE='https://johanstrydom.com'
PINS={'traffic.csv.gz': {'sha256': '2db1cc2ac20f6a0cce917207b9f8c8d310119393b67e7bdc94dcb1adfa19d480', 'url': 'https://johanstrydom.com/stats19-evidence/2db1cc2ac20f6a0cce917207b9f8c8d310119393b67e7bdc94dcb1adfa19d480.csv.gz'}, 'audit-20mph-monitoring.py': {'sha256': 'c1e3e735c778aa936aad13f4d74cb12ad369eac2ea78a6495dd4d6fefe623797', 'url': 'https://johanstrydom.com/research/20mph/v1/audit-20mph-monitoring.py'}, 'Vehicle_speed_monitoring_data_2025.ods': {'sha256': '32d07beea649ab3316abaeb7602d4f5dffd7385a4801b133c32d911d06e0c08c', 'url': 'https://tfw.wales/sites/default/files/2025-12/Vehicle_speed_monitoring_data_2025.ods'}, 'Vehicle_journey_times_Feb23_to_Nov24.ods': {'sha256': 'c73a995c62c69e7d4c35c6d235805d5e51453f891d61d1aeea9f8696a04708d4', 'url': 'https://tfw.wales/sites/default/files/2025-07/Vehicle_journey_times_Feb23_to_Nov24.ods'}, 'collisions_national_source.csv.gz': {'sha256': 'c600d84ff58c3003f788dfccedb3750935bb1c52e3aa7b74717b36a707c14171', 'url': 'https://johanstrydom.com/research/20mph/v2/inputs/collisions_national_source.csv.gz'}, 'casualties_national_source.csv.gz': {'sha256': '1f9fc4c0ac00cdd4d27a43c6efa1687ea53b2a0ce7d101e0fc4f7a8cd346bda9', 'url': 'https://johanstrydom.com/research/20mph/v2/inputs/casualties_national_source.csv.gz'}}

def checked(name,b):
 raw=gzip.decompress(b) if name.endswith('.gz') else b
 if hashlib.sha256(raw).hexdigest()!=PINS[name]['sha256']:raise ValueError('Input changed: '+name)
 return raw

def cells(b):
 rows=list(csv.DictReader(io.StringIO(b.decode('utf-8-sig'))));out={}
 for r in rows:
  assert r['Area_reference']=='W92000004' and r['By highways agency_reference']=='2'
  if 'Type of Vehicle_reference' in r:assert r['Type of Vehicle_reference']=='12'
  k=(r['Date_reference'],r['Speed limit_reference'],r['Severity_reference'])
  assert k not in out,'Duplicate source cell';out[k]=(int(r['Data values']),r['Notes'])
 assert len(out)==2280
 for period in {k[0] for k in out}:
  for sev in ['1','2','3','4','5']:assert sum(out[period,s,sev][0] for s in ['010','020','030','040','050','060','070'])==out[period,'Total',sev][0]
  for s in ['010','020','030','040','050','060','070','Total']:
   get=lambda v:out[period,s,v][0]
   assert get('1')+get('2')==get('4') and get('4')+get('3')==get('5')
 for y in range(2015,2026):
  for s in ['010','020','030','040','050','060','070','Total']:
   for sev in ['1','2','3','4','5']:assert sum(out[f'{y}_{q}',s,sev][0] for q in range(1,5))==out[f'{y}_5',s,sev][0]
 return out

def calculate(raw,inputs):
 indexes={m:cells(raw[m+'_national_source.csv.gz']) for m in ['collisions','casualties']};annual=[];quarterly=[]
 for period in sorted({k[0] for k in indexes['collisions']}):
  y,q=map(int,period.split('_'));r={'year':y,'quarter':q,'label':str(y) if q==5 else f'{y} Q{q}','provisional':False}
  for m,idx in indexes.items():
   for name,speed in [('all','Total'),('20','020'),('30','030')]:r[m+'_'+name]=idx[period,speed,'5'][0]
   r[m+'_20_30']=r[m+'_20']+r[m+'_30'];r[m+'_other']=r[m+'_all']-r[m+'_20_30'];r[m+'_notes']=idx[period,'Total','5'][1]
   r['provisional']|=any('Provisional' in idx[period,s,'5'][1] for s in ['Total','020','030'])
  (annual if q==5 else quarterly).append(r)
 assert len(annual)==11 and len(quarterly)==46 and sum(r['provisional'] for r in quarterly)==2
 mileage={}
 for r in csv.DictReader(io.StringIO(raw['traffic.csv.gz'].decode())):
  if r['region_ons_code']=='W92000004' and 2015<=int(r['year'])<=2025:
   assert int(r['year']) not in mileage;mileage[int(r['year'])]=float(r['all_motor_vehicles'])
 assert len(mileage)==11
 for r in annual:
  r['motor_vehicle_miles']=mileage[r['year']];assert r['motor_vehicle_miles']>0
  for m in indexes:r[m+'_per_100k_motor_miles']=r[m+'_all']/r['motor_vehicle_miles']*100000
 spec=importlib.util.spec_from_file_location('monitoring',inputs/'audit-20mph-monitoring.py');mon=importlib.util.module_from_spec(spec);spec.loader.exec_module(mon)
 speed=mon.read(inputs/'Vehicle_speed_monitoring_data_2025.ods');speeds=[]
 for period in ['Summer 2023','Winter 2023/24','Summer 2024','Winter 2024/25','Summer 2025']:
  r={'period':period}
  for table,key,mult in [('Table_1','at_or_below_24_pct',100),('Table_2','p85_mph',1),('Table_3','mean_mph',1)]:r[key]=round(next(row for row in speed[table] if row[0]['text']==period)[1]['value']*mult,1)
  speeds.append(r)
 journey=mon.read(inputs/'Vehicle_journey_times_Feb23_to_Nov24.ods');journeys=[];checks=0
 for row in journey['Table_2']:
  if len(row)<10 or row[4]['seconds'] is None:continue
  ids=[row[i]['text'] for i in range(4)];v=next(r for r in journey['Table_4'] if [c['text'] for c in r[:4]]==ids);r=dict(zip(['route','description','peak','direction'],ids))
  for src,key in [(row,'mean'),(v,'spread')]:
   a,b,c=[round(src[i]['seconds']) for i in [4,5,6]]
   for i,val in zip([7,8,9],[b-a,c-b,c-a]):assert abs(src[i]['seconds']-val)<.01;checks+=1
   for p,val in zip(['spring2023','spring2024','autumn2024'],[a,b,c]):r[key+'_'+p]=val
   r[key+'_spring_change']=b-a;r[key+'_autumn_change']=c-a
  for season,year,p in [('Spring','2023','spring2023'),('Spring','2024','spring2024'),('Autumn','2024','autumn2024')]:
   match=lambda t:next(a for a in journey[t] if len(a)>6 and [a[i]['text'] for i in [0,2,3,4,5]]==[r['route'],r['peak'],season,year,r['direction']])
   a=match('Table_1');b=match('Table_3');assert round(a[6]['seconds'])==r['mean_'+p]
   r['p05_'+p]=round(b[6]['seconds']);r['p95_'+p]=round(b[7]['seconds']);assert r['p95_'+p]-r['p05_'+p]==r['spread_'+p]==round(b[8]['seconds'])
  journeys.append(r)
 assert len(journeys)==60 and len({(r['route'],r['peak'],r['direction']) for r in journeys})==60 and checks==360
 summary={}
 for w in ['spring','autumn']:
  vals=[r['mean_'+w+'_change'] for r in journeys];summary[w]={'increased':sum(v>0 for v in vals),'unchanged':sum(v==0 for v in vals),'decreased':sum(v<0 for v in vals),'increases_up_to_120_seconds':sum(0<v<=120 for v in vals),'median_seconds':statistics.median(vals),'min_seconds':min(vals),'max_seconds':max(vals)}
 assert annual[-1]['collisions_all']==3005 and annual[-1]['casualties_all']==4079
 assert annual[-1]['collisions_20_30']==1470 and annual[-1]['casualties_20_30']==1797
 assert quarterly[-1]['collisions_all']==638 and quarterly[-1]['casualties_all']==856
 return {'release':'20mph-phase2-v2','asOf':'2026-11-04','statsWalesRelease':'2026-10-29','annual':annual,'quarterly':quarterly,'speeds':speeds,'journeys':journeys,'journeySummary':summary,'checks':{'nationalSourceCells':4560,'quarterlyPeriods':46,'annualPeriods':11,'sourceSubtotalsReconcile':True,'journeyDifferenceChecks':checks,'journeyPercentileChecks':180},'inputs':PINS}

def output(data,out):
 out.mkdir(parents=True,exist_ok=True);(out/'pilot.json').write_text(json.dumps(data,indent=2)+'\n')
 for key in ['quarterly','annual','speeds','journeys']:
  with (out/(key+'.csv')).open('w',newline='') as f:
   w=csv.DictWriter(f,fieldnames=list(data[key][0]));w.writeheader();w.writerows(data[key])
 hashes={n:hashlib.sha256((out/n).read_bytes()).hexdigest() for n in ['pilot.json','quarterly.csv','annual.csv','speeds.csv','journeys.csv']};(out/'chart-checksums.json').write_text(json.dumps(hashes,indent=2)+'\n');return hashes

def main():
 p=argparse.ArgumentParser();root=pathlib.Path(__file__).resolve().parents[1];p.add_argument('--input-dir',type=pathlib.Path,default=root/'research-archive/20mph/v2');p.add_argument('--output-dir',type=pathlib.Path,default=root/'public/research/20mph/v2');p.add_argument('--download',action='store_true');a=p.parse_args();a.input_dir.mkdir(parents=True,exist_ok=True);raw={}
 for n,pin in PINS.items():
  f=a.input_dir/n
  if not f.exists() and a.download:
   try:b=urllib.request.urlopen(pin['url'],timeout=90).read()
   except urllib.error.HTTPError as e:raise SystemExit(f'Download refused (HTTP {e.code}): {pin["url"]}. Save this input through your browser as {f}, then rerun. See README.txt for the complete manual input list.')
   checked(n,b);f.write_bytes(b)
  raw[n]=checked(n,f.read_bytes())
 d=calculate(raw,a.input_dir);print(json.dumps({'checks':d['checks'],'outputs':output(d,a.output_dir),'journeySummary':d['journeySummary']},indent=2))
if __name__=='__main__':main()
