def gamma_values(gamma=0.9, n=50):
    return [gamma**i for i in range(1, n)]

plot(gamma_values())
title('gamma value in regard with t. gamma=0.9, n=50')
xlabel('time t')
ylabel('gamma value')
grid()

DELTA = 4



data = {'mean sqaured error': [],
        'mean absolute error': [],
        'huber': [],
        'pseudo huber': [],
        'my huber': []}
x = np.arange(-4, 4.1, 0.1)

def anderson_huber(delta, value):
    """
    scipy.special.huber의 결과값과 동일하게 나옴.
    """

    if abs(value) <= delta:
        return 1/2 * value**2
    return delta * (abs(value) - 1/2 * delta)

for y in x:
    mse = mean_squared_error([0], [y])
    mae = mean_absolute_error([0], [y])
    he = huber(DELTA, 0-y)
    my_he = anderson_huber(DELTA, 0-y)
    phe = pseudo_huber(DELTA, 0-y)

    data['mean sqaured error'].append(mse)
    data['mean absolute error'].append(mae)
    data['huber'].append(he)
    data['pseudo huber'].append(phe)
    data['my huber'].append(my_he)


del data['pseudo huber']
data = pd.DataFrame(data, index=x)
data.plot()

sudo pip3 install pygame

git clone https://github.com/ntasfi/PyGame-Learning-Environment.git
cd PyGame-Learning-Environment/
sudo pip3 install -e .

git clone https://github.com/lusob/gym-ple.git
cd gym-ple/
sudo pip3 install -e .

class ReplayMemory(object):
    def __init__(self, capacity=REPLAY_MEMORY):
        self.capacity = capacity
        self.memory = deque(maxlen=self.capacity)
        self.Transition = namedtuple('Transition', ('state', 'action', 'reward', 'next_state'))
        self._available = False

    def put(self, state: np.array, action: torch.LongTensor, reward: np.array, next_state: np.array):
        """
        저장시 모두 Torch Tensor로 변경해준다음에 저장을 합니다.
        action은 select_action()함수에서부터 LongTensor로 리턴해주기 때문에,
        여기서 변경해줄필요는 없음
        """
        state = torch.FloatTensor(state)
        reward = torch.FloatTensor([reward])
        if next_state is not None:
            next_state = torch.FloatTensor(next_state)
        transition = self.Transition(state=state, action=action, reward=reward, next_state=next_state)
        self.memory.append(transition)

    def sample(self, batch_size):
        transitions = sample(self.memory, batch_size)
        return self.Transition(*(zip(*transitions)))

class DQN(nn.Module):
    def __init__(self, n_action):
        super(DQN, self).__init__()
        self.n_action = n_action

        self.conv1 = nn.Conv2d(4, 32, kernel_size=8, stride=4, padding=0)  # (In Channel, Out Channel, ...)
        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0)
        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0)

        self.affine1 = nn.Linear(3136, 512)
        self.affine2 = nn.Linear(512, self.n_action)

    def forward(self, x):
        h = F.relu(self.conv1(x))
        h = F.relu(self.conv2(h))
        h = F.relu(self.conv3(h))

        h = F.relu(self.affine1(h.view(h.size(0), -1)))
        h = self.affine2(h)
        return h

def train(self, gamma: float = 0.99, mode: str = 'rgb_array'):
    while True:
        states = self.get_initial_states()
        losses = []
        checkpoint_flag = False
        target_update_flag = False
        play_steps = 0

        reward = 0
        done = False
        while True:
            # Get Action
            action: torch.LongTensor = self.select_action(states)

            for _ in range(self.frame_skipping):
                # step 에서 나온 observation은 버림
                observation, reward, done, info = self.env.step(action[0, 0])
                next_state = self.env.get_screen()
                self.add_state(next_state)

                if done:
                    break

            # Store the infomation in Replay Memory
            next_states = self.recent_states()
            if done:
                self.replay.put(states, action, reward, None)
            else:
                self.replay.put(states, action, reward, next_states)

            # Change States
            states = next_states

            # Optimize
            if self.replay.is_available():
                loss, reward_sum, q_mean, target_mean = self.optimize(gamma)
                losses.append(loss[0])

            if done:
                break

            # Increase step
            self.step += 1
            play_steps += 1

            # Target Update
            if self.step % TARGET_UPDATE_INTERVAL == 0:
                self._target_update()
                target_update_flag = True


def optimize(self, gamma: float):

    # Get Sample
    transitions = self.replay.sample(BATCH_SIZE)

    # Mask
    non_final_mask = torch.ByteTensor(list(map(lambda ns: ns is not None, transitions.next_state))).cuda()
    final_mask = 1 - non_final_mask

    state_batch: Variable = Variable(torch.cat(transitions.state).cuda())
    action_batch: Variable = Variable(torch.cat(transitions.action).cuda())
    reward_batch: Variable = Variable(torch.cat(transitions.reward).cuda())
    non_final_next_state_batch = Variable(torch.cat([ns for ns in transitions.next_state if ns is not None]).cuda())
    non_final_next_state_batch.volatile = True

    # Reshape States and Next States
    state_batch = state_batch.view([BATCH_SIZE, self.action_repeat, self.env.width, self.env.height])
    non_final_next_state_batch = non_final_next_state_batch.view(
        [-1, self.action_repeat, self.env.width, self.env.height])
    non_final_next_state_batch.volatile = True

    # Clipping Reward between -2 and 2
    reward_batch.data.clamp_(-1, 1)

    # Predict by DQN Model
    q_pred = self.dqn(state_batch)
    q_values = q_pred.gather(1, action_batch)

    # Predict by Target Model
    target_values = Variable(torch.zeros(BATCH_SIZE, 1).cuda())
    target_pred = self.target(non_final_next_state_batch)
    target_values[non_final_mask] = reward_batch[non_final_mask] + target_pred.max(1)[0] * gamma
    target_values[final_mask] = reward_batch[final_mask]

    loss = F.smooth_l1_loss(q_values, target_values)
    # loss = torch.mean((target_values - q_values) ** 2)
    self.optimizer.zero_grad()
    loss.backward()

    if self.clip:
        for param in self.dqn.parameters():
            param.grad.data.clamp_(-1, 1)
    self.optimizer.step()

    reward_score = int(torch.sum(reward_batch).data.cpu().numpy()[0])
    q_mean = torch.sum(q_pred, 0).data.cpu().numpy()[0]
    target_mean = torch.sum(target_pred, 0).data.cpu().numpy()[0]

    return loss.data.cpu().numpy(), reward_score, q_mean, target_mean

mkdir frames
ffmpeg -i video.mp4 -qscale:v 2  -r 25 'frames/frame-%03d.jpg'
cd frames
convert -delay 4 -loop 0 *.jpg myimage.gif