Pythonで学ぶトークン化：語彙、one-hot、バイトペアエンコーディング

from collections import Counter

text = "the cat sat on the mat"

# 1. Build a vocabulary: every distinct word gets an id.
words = text.split()
vocab = {word: i for i, word in enumerate(sorted(set(words)))}
print("vocab:", vocab)
print("ids:", [vocab[w] for w in words])

# 2. One-hot encoding: a vector with a single 1 at the word's id.
def one_hot(word):
    return [1 if vocab[word] == i else 0 for i in range(len(vocab))]
print("one-hot 'cat':", one_hot("cat"))

# 3. Byte pair encoding: repeatedly merge the most frequent pair of symbols.
tokens = list("banana")
for _ in range(2):
    pairs = Counter(zip(tokens, tokens[1:]))
    best = max(pairs, key=pairs.get)
    merged, i = [], 0
    while i < len(tokens):
        if i + 1 < len(tokens) and (tokens[i], tokens[i + 1]) == best:
            merged.append(tokens[i] + tokens[i + 1])
            i += 2
        else:
            merged.append(tokens[i])
            i += 1
    tokens = merged
    print("merge", best, "->", tokens)