Python实现蒙特卡洛算法解决冰湖问题

import math
from collections import defaultdict
import gymnasium as gym
from el_agent import ELAgent
from frozen_lake_util import show_q_value

class MontCarloAgent(ELAgent):

    def __init__(self, epsilon):
        super().__init__(epsilon)

    def learn(self, env, episode_count=100, gama=0.0,
              render=False, report_interval=50):
        self.init_log()
        self.Q = defaultdict(lambda: [0] * len(actions))
        N = defaultdict(lambda :[0] * len(actions))
        actions = list(range(env.action_space.n))

        for e in range(episode_count):
            s = env.reset()[0]
            done = False
            # 1.进行到回合结束为止
            experience = []
            while not done:
                if render:
                    env.render()
                a = self.policy(s, actions)
                n_state, reward, terminated, truncated, info = env.step(a)
                experience.append({'state': s, 'action': a, 'reward': reward})
                s = n_state
                done = terminated or truncated
            else:
                self.log(reward)

            # 2. 估计各种状态和行动
            for i, x in enumerate(experience):
                s, a = x['state'], x['action']
                # 计算状态s对应的折现值
                G, t = 0, 0
                for j in range(i, len(experience)):
                    G += math.pow(gama, t) * experience[j]['reward']
                    t += 1
                N[s][a] += 1  # s,a的对数
                alpha = 1 / N[s][a]
                self.Q[s][a] += alpha * (G - self.Q[s][a])
            if e != 0 and e % report_interval ==0:
                self.show_reward_log(experience=e)

def train():
    agent = MontCarloAgent(epsilon=0.1)
    env = gym.make('FrozenLake-v1', render_mode='human', is_slippery=False)
    agent.learn(env, episode_count=500)
    show_q_value(agent.Q)
    agent.show_reward_log()

if __name__ == '__main__':
    train()