"""Synthetic file/segment language-selection evaluator, not ASR or model comparison."""
import json,copy
ALLOWED={'fa','en'};KNOWN={'fa','en','ar'}
def validate(rows,predictions):
    keys=set()
    for r in rows:
        k=(r['file'],r['unit'],r['id'])
        if k in keys or r['unit']not in ['file','segment']:raise ValueError('unique declared file/unit/id')
        keys.add(k)
        if r['class']not in ['single_language','no_speech','mixed_excluded']:raise ValueError('declared reference class')
        if r['class']=='single_language'and r['truth']not in KNOWN:raise ValueError('known reference code required')
        if r['class']!='single_language'and r['truth']is not None:raise ValueError('mixed/no-speech is not a single truth label')
        if r['unit']=='file'and(r['start']is not None or r['end']is not None):raise ValueError('file reference has no invented segment times')
        if r['unit']=='segment'and not(0<=r['start']<r['end']):raise ValueError('positive reference segment bounds')
    if set(predictions)!=keys:raise ValueError('prediction keys must match exact file/unit/id reference')
    if any(x not in ALLOWED|{None}for x in predictions.values()):raise ValueError('prediction outside declared allowed codes')
def evaluate(rows,predictions):
    validate(rows,predictions);out={}
    for unit in ['file','segment']:
        subset=[r for r in rows if r['unit']==unit];eligible=[r for r in subset if r['class']=='single_language'and r['truth']in ALLOWED]
        counts={'reference_rows':len(subset),'eligible_in_allowed':len(eligible),'correct':0,'wrong':0,'abstained_eligible':0,'answered_eligible':0,'no_speech_reference':0,'outside_allowed_reference':0,'mixed_excluded_reference':0,'noneligible_answered':0,'noneligible_abstained':0}
        for r in subset:
            p=predictions[(r['file'],r['unit'],r['id'])]
            if r in eligible:
                if p is None:counts['abstained_eligible']+=1
                else:counts['answered_eligible']+=1;counts['correct'if p==r['truth']else'wrong']+=1
            else:
                if r['class']=='no_speech':counts['no_speech_reference']+=1
                elif r['class']=='mixed_excluded':counts['mixed_excluded_reference']+=1
                else:counts['outside_allowed_reference']+=1
                counts['noneligible_abstained'if p is None else'noneligible_answered']+=1
        n=counts['eligible_in_allowed'];a=counts['answered_eligible']
        counts['coverage_fraction']=f'{a}/{n}'if n else None;counts['correct_over_all_eligible']=f"{counts['correct']}/{n}"if n else None;counts['accuracy_among_answered']=f"{counts['correct']}/{a}"if a else None
        out[unit]=counts
    return out
def demo():
    rows=[]
    for file,cls,truth in [('f1','single_language','fa'),('f2','single_language','en'),('f3','no_speech',None),('f4','single_language','ar'),('f5','mixed_excluded',None)]:
        rows.append({'file':file,'unit':'file','id':'whole','start':None,'end':None,'class':cls,'truth':truth})
    for id,start,end,cls,truth in [('s1',0,2,'single_language','fa'),('s2',2,4,'single_language','en'),('s3',4,5,'no_speech',None)]:
        rows.append({'file':'f5','unit':'segment','id':id,'start':start,'end':end,'class':cls,'truth':truth})
    manual={(r['file'],r['unit'],r['id']):'fa'for r in rows}
    auto={k:p for k,p in zip(manual,['fa','en',None,'en','fa','fa','en',None])}
    m=evaluate(rows,manual);a=evaluate(rows,auto)
    assert m['file']['correct_over_all_eligible']=='1/2'and a['file']['correct_over_all_eligible']=='2/2'
    assert m['segment']['correct_over_all_eligible']=='1/2'and a['segment']['correct_over_all_eligible']=='2/2'
    assert a['file']['outside_allowed_reference']==1 and a['file']['noneligible_answered']==2
    abstain=dict(auto);abstain[('f2','file','whole')]=None;b=evaluate(rows,abstain);assert b['file']['coverage_fraction']=='1/2'and b['file']['accuracy_among_answered']=='1/1'and b['file']['correct_over_all_eligible']=='1/2'
    errors={}
    bad=dict(auto);bad[('wrong-file','file','whole')]=bad.pop(('f1','file','whole'))
    for name,rr,pp in [('wrong_file',rows,bad),('wrong_prediction_code',rows,{**auto,('f1','file','whole'):'zz'}),('segment_as_file',[{**r,'unit':'file'}if r['unit']=='segment'else r for r in rows],auto)]:
        try:evaluate(rr,pp)
        except(ValueError,TypeError)as e:errors[name]=str(e)
        else:raise AssertionError('mutation accepted')
    return {'authored_reference_not_labeled_audio':rows,'authored_manual_predictions':[{'key':list(k),'prediction':v}for k,v in manual.items()],'authored_auto_predictions':[{'key':list(k),'prediction':v}for k,v in auto.items()],'manual_policy':m,'auto_policy':a,'abstention_example':b,'invalid_mutations_rejected':errors,'policy':'fa/en allowed; ar is known but outside allowed, no_speech unknown is distinct; mixed whole-file excluded under explicit no-single-label policy, its reference segments scored separately. Noneligible abstention is reported, never counted as correct language.','real_audio_or_engine_test':False,'not_evidence_of_actual_accuracy_or_product_language_support':True}
if __name__=='__main__':print(json.dumps(demo(),ensure_ascii=False,indent=2))