Python으로 배우는 강화 학습: 가치 반복과 Q-러닝

import random

# A corridor of 5 cells: start in cell 0, reward +10 for reaching cell 4.
N, GOAL, GAMMA = 5, 4, 0.9

# Value iteration: a cell is worth its reward plus the discounted best neighbour.
values = [0.0] * N
for _ in range(50):
    values = [0.0 if s == GOAL else max((10 if n == GOAL else 0) + GAMMA * values[n]
              for n in (max(0, s - 1), min(N - 1, s + 1))) for s in range(N)]
print("state values:", [round(v, 2) for v in values])

# Q-learning learns the same thing from trial and error.
random.seed(0)
Q = [[0.0, 0.0] for _ in range(N)]   # actions: 0 = left, 1 = right
for episode in range(200):
    s = 0
    while s != GOAL:
        a = random.randrange(2) if random.random() < 0.2 else Q[s].index(max(Q[s]))
        n = max(0, s - 1) if a == 0 else min(N - 1, s + 1)
        reward = 10 if n == GOAL else 0
        target = reward + (0 if n == GOAL else GAMMA * max(Q[n]))
        Q[s][a] += 0.5 * (target - Q[s][a])   # move the estimate toward the target
        s = n
print("best action per cell:", ["right" if q[1] >= q[0] else "left" for q in Q[:GOAL]])
print("Q(0, right):", round(Q[0][1], 2))