"""Tiny character-based BPE with authored Persian word frequencies."""
from collections import Counter
import json
CORPUS={'کار':4,'کاربر':2,'بار':3,'زار':1}
def merge(parts,pair):
    out=[];i=0
    while i<len(parts):
        if i+1<len(parts) and (parts[i],parts[i+1])==pair:out.append(parts[i]+parts[i+1]);i+=2
        else:out.append(parts[i]);i+=1
    return out
def train(steps=3):
    splits={w:list(w) for w in CORPUS};history=[]
    for _ in range(steps):
        counts=Counter()
        for w,parts in splits.items():
            for i in range(len(parts)-1):counts[(parts[i],parts[i+1])]+=CORPUS[w]
        if not counts:break
        pair=min(counts,key=lambda p:(-counts[p],p))
        history.append({'pair':list(pair),'weighted_occurrences':counts[pair]})
        splits={w:merge(parts,pair) for w,parts in splits.items()}
    return history,splits
def encode(word,history):
    alphabet=set(''.join(CORPUS))
    if any(c not in alphabet for c in word):raise ValueError('unknown character in this toy contract')
    parts=list(word)
    for item in history:parts=merge(parts,tuple(item['pair']))
    return parts
if __name__=='__main__':
    h,s=train()
    assert [x['pair'] for x in h]==[['ا','ر'],['ک','ار'],['ب','ار']]
    assert [x['weighted_occurrences'] for x in h]==[10,6,3]
    assert encode('کاربار',h)==['کار','بار'] and encode('کارزار',h)==['کار','ز','ار']
    assert ''.join(encode('کاربار',h))=='کاربار'
    try:encode('نام',h)
    except ValueError:pass
    else:raise AssertionError('unknown character accepted')
    print(json.dumps({'corpus':CORPUS,'three_merges':h,'training_splits':s,'new_word_encoding':encode('کاربار',h),'real_LLM_tokenizer':False},ensure_ascii=False,indent=2))