"""Shared deterministic PubMedQA evaluator; invalid predictions count as errors."""
import argparse,csv,json,re
from pathlib import Path
import numpy as np
LABELS=('yes','no','maybe')
def parse_answer(text):
    s=text.strip().lower()
    m=re.fullmatch(r'(yes|no|maybe)[.!]?',s)
    return m.group(1) if m else 'invalid'
def metrics(y,p):
    cm=np.zeros((3,4),dtype=int)
    for a,b in zip(y,p): cm[LABELS.index(a),LABELS.index(b) if b in LABELS else 3]+=1
    tp=np.diag(cm[:,:3]); precision=np.divide(tp,cm[:,:3].sum(0),out=np.zeros(3),where=cm[:,:3].sum(0)>0)
    recall=np.divide(tp,cm.sum(1),out=np.zeros(3),where=cm.sum(1)>0)
    f1=np.divide(2*precision*recall,precision+recall,out=np.zeros(3),where=precision+recall>0)
    return {'accuracy':float(tp.sum()/len(y)),'macro_f1':float(f1.mean()),'per_class':{k:{'precision':float(precision[i]),'recall':float(recall[i]),'f1':float(f1[i]),'support':int(cm[i].sum())} for i,k in enumerate(LABELS)},'confusion_matrix':cm.tolist(),'invalid':int(cm[:,3].sum()),'n':len(y)}
def load(path):
    rows=[json.loads(x) for x in Path(path).read_text().splitlines() if x.strip()] if path.endswith('.jsonl') else list(csv.DictReader(open(path)))
    assert len({str(r['id']) for r in rows})==len(rows)
    for r in rows:
        r['prediction']=parse_answer(r['raw_output'])
        assert r['label'] in LABELS
    return {str(r['id']):r for r in rows}
def evaluate(before,after,output,n_boot=10000,seed=20261029):
    a,b=load(before),load(after); assert a.keys()==b.keys()
    ids=sorted(a); y=np.array([a[k]['label'] for k in ids]); p=np.array([a[k]['prediction'] for k in ids]); q=np.array([b[k]['prediction'] for k in ids]); assert all(a[k]['label']==b[k]['label'] for k in ids)
    result={'baseline':metrics(y,p),'finetuned':metrics(y,q),'bootstrap':{'resamples':n_boot,'seed':seed,'method':'paired iid percentile'}}
    rng=np.random.default_rng(seed); draws=np.empty((n_boot,2,2))
    for j in range(n_boot):
        ix=rng.integers(0,len(y),len(y))
        for k,pred in enumerate((p,q)):
            m=metrics(y[ix],pred[ix]); draws[j,k]=[m['accuracy'],m['macro_f1']]
    for k,metric in enumerate(('accuracy','macro_f1')):
        result['bootstrap'][metric]={'baseline_ci95':np.quantile(draws[:,0,k],[.025,.975]).tolist(),'finetuned_ci95':np.quantile(draws[:,1,k],[.025,.975]).tolist(),'delta_ci95':np.quantile(draws[:,1,k]-draws[:,0,k],[.025,.975]).tolist(),'delta':result['finetuned'][metric]-result['baseline'][metric]}
    Path(output).write_text(json.dumps(result,ensure_ascii=False,indent=2)); return result
if __name__=='__main__':
    p=argparse.ArgumentParser();p.add_argument('--before',required=True);p.add_argument('--after',required=True);p.add_argument('--output',required=True);p.add_argument('--resamples',type=int,default=10000);a=p.parse_args();print(json.dumps(evaluate(a.before,a.after,a.output,a.resamples),indent=2))