passport/run.py at master · deconlabs/passport · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
import numpy as np
import random
import sys
import pickle
import os

from agent import Agent
from arguments import argparser
from env import Env


def distribute_asset(env, n_agent):
    # probs=np.random.normal(total_asset/n_agent, 3, n_agent)
    probs = np.random.pareto(2, n_agent)
    probs /= np.sum(probs)
    probs = sorted(probs)[::-1]

    for i in range(n_agent):
        env.agent_assets[i] = probs[i]


def run(env, agents, args):
    res_returns = []
    res_costs = []
    res_rewards = []
    res_actions = []
    res_highests = []
    res_total_beta_lists = []
    res_likes = []
    detail_beta_lists = []

    """per episode"""
    for episode in range(args.n_episode + 1):
        actions = []
        for agent in agents:
            actions.append(agent.get_action())  # 확률론적

        """per step"""
        # just a one step.
        _, rewards, _, info = env.step(actions)

        for idx, agent in enumerate(agents):
            agent.learn(actions[idx], rewards[idx])

        # review_ratio = info['review_ratio']
        returns = info['returns']
        costs = info['costs']
        likes = info['likes']

        highests = [agent.get_action(deterministic=True) for agent in agents]
        total_beta_lists = [agent.beta_table for agent in agents]

        if episode % args.record_term_1 == 0:
            res_returns.append(returns)
            res_costs.append(costs)
            res_rewards.append(rewards)
            res_actions.append(actions)
            res_highests.append(highests)
            res_total_beta_lists.append(total_beta_lists)
            res_likes.append(likes)

        if episode % args.record_term_2 == 0:
            detail_beta_lists.append(total_beta_lists)

    return (
        res_returns,
        res_costs,
        res_rewards,
        res_actions,
        res_highests,
        res_total_beta_lists,
        res_likes,
        detail_beta_lists
    )


if __name__ == '__main__':
    my_args = sys.argv
    print(my_args)
    args = argparser()

    """set random seeds"""
    np.random.seed(950327)
    random.seed(950327)

    all_returns = []
    all_costs = []
    all_rewards = []
    all_actions = []
    all_highests = []
    all_total_beta_lists = []
    all_likes = []
    all_details = []

    for i in range(args.n_average):
        env = Env(args)
        distribute_asset(env, args.n_agent)
        agents = [Agent(env.action_space, args) for i in range(args.n_agent)]

        """run"""
        returns, costs, rewards, actions, highests, total_beta_lists, likes, details = run(env, agents, args)

        all_returns.append(returns)
        all_costs.append(costs)
        all_rewards.append(rewards)
        all_actions.append(actions)
        all_highests.append(highests)
        all_total_beta_lists.append(total_beta_lists)
        all_likes.append(likes)
        all_details.append(details)

        print("loop", i, "done")

    """pickle files save"""
    meta_dict = {
        'all_returns': all_returns,
        'all_costs': all_costs,
        'all_rewards': all_rewards,
        'all_actions': all_actions,
        'all_highests': all_highests,
        'all_total_beta_lists': all_total_beta_lists,
        'all_likes': all_likes,
        'all_details_total_beta_lists': all_details
    }

    filename = "./data/{}_{}_{}_{}_{}.pkl".format(
        my_args[1][2:], my_args[2][2:], my_args[3][2:], my_args[4][2:], my_args[5][2:])
    if not os.path.exists('./data'):
        os.mkdir('./data')
    with open(filename, 'wb') as f:
        pickle.dump(meta_dict, f)