Pythonで学ぶAttentionとSelf-Attention

import math

def softmax(row):
    exps = [math.exp(v - max(row)) for v in row]
    return [e / sum(exps) for e in exps]

# Three tokens, each with a query, key and value vector of size 2.
Q = [[1, 0], [0, 1], [1, 1]]
K = [[1, 0], [0, 1], [1, 1]]
V = [[10, 0], [0, 10], [5, 5]]
d = len(K[0])

def attention(causal=False):
    out = []
    for i, q in enumerate(Q):
        # Score: how well this query matches every key, scaled by sqrt(d).
        scores = [sum(a * b for a, b in zip(q, k)) / math.sqrt(d) for k in K]
        if causal:  # a token may only look at itself and earlier tokens
            scores = [s if j <= i else -1e9 for j, s in enumerate(scores)]
        weights = softmax(scores)
        out.append([round(sum(w * v[c] for w, v in zip(weights, V)), 2) for c in range(2)])
    return out

print("attention:", attention())
print("causal attention:", attention(causal=True))