rltrading-program.py

# -*- coding: utf-8 -*-
"""Copy of Untitled4.ipynb

Automatically generated by Colaboratory.

[Link to Colab's page](https://drive.google.com/file/d/1P8mhR8AsZ9Ol3bz1gbar_P0VHDfQm_-2/view?usp=sharing)

##Course Project. COMP-767: Reinforcement Learning Learning, Winter 2018

###Data Preparation:
"""

# To reset the virtual machine: ! kill -9 -1
! git -C ReinforcementLearningProject pull || git clone https://github.com/madarez/ReinforcementLearningProject

"""###Libraries:"""

from ast import literal_eval
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from datetime import date, datetime
import numpy.linalg as linalg

def load(symb):
  with open('ReinforcementLearningProject/historical_data/%s.mat' % symb) as f:
    list_data = literal_eval(f.read())

  print('Historical data of %s symbol' % symb)
  df = pd.DataFrame(list_data,columns=['MTS','OPEN','CLOSE','HIGH','LOW','VOLUME'])
  #df.set_index('MTS', inplace=True)
  date=pd.to_datetime(df['MTS'],unit='ms')
  df = df / 1e6 # scale down Bitcoins to Satoshis and time
  #df['YEAR']=date.dt.year-2010
  df['MONTH']=date.dt.month
  #df['DAY']=date.dt.day
  df['WEEKDAY']=date.dt.dayofweek
  
  return df

"""###Building environments and algorithms:"""

symb = ['BTCUSD', 'ETHUSD', 'EOSUSD', 'XRPUSD', 'BCHUSD', 'NEOUSD', 'LTCUSD',
        'IOTUSD', 'XMRUSD', 'ETCUSD', 'OMGUSD', 'DSHUSD', 'ZECUSD', 'TRXUSD',
        'BTGUSD', 'ETPUSD', 'SANUSD', 'BFTUSD', 'BATUSD', 'QTMUSD', 'GNTUSD',
        'ZRXUSD', 'EDOUSD', 'REPUSD', 'ELFUSD', 'SPKUSD', 'TNBUSD', 'SNTUSD',
        'QSHUSD', 'WAXUSD', 'FUNUSD', 'DAIUSD', 'YYWUSD', 'DATUSD', 'MNAUSD',
        'LRCUSD', 'AVTUSD', 'IOSUSD', 'MTNUSD', 'AIDUSD', 'NECUSD', 'RDNUSD',
        'AGIUSD', 'RCNUSD', 'SNGUSD', 'RLCUSD', 'RRTUSD', 'AIOUSD', 'CFIUSD',
        'ODEUSD', 'REQUSD']

class Env(object):
  def __init__(self, capital = 0.01, delay = 25):
    self.capital = capital # $0.01 default initial capital
    self.satoshi = 0
    self.timesteps = 0
    self.delay = delay # 25*15 min ~ 6 hours by default delay of each Tx
    self.satoshi_queue = np.zeros(delay)
    self.capital_queue = np.zeros(delay)

  def reset(self):
    np.roll(self.satoshi_queue, - self.timesteps) # to continue receiving the rewards
    np.roll(self.capital_queue, - self.timesteps) # to continue receiving the rewards
    self.timesteps = 0
    #self.data = load(np.random.choice(symb))
    self.data = load('BTCUSD')
    state = self.state_maker()
    return state

  def state_maker(self):
    f0 = self.satoshi > 0 # nonempty wallet
    f1 = self.capital > self.data.loc[self.timesteps,'OPEN'] # affordability
    f2 = self.data.loc[self.timesteps,'OPEN'] < (self.data.loc[self.timesteps,'LOW'] * 1.5) # average
    f3 = self.data.loc[self.timesteps,'CLOSE'] < (self.data.loc[self.timesteps,'LOW'] * 1.5) # average
    f4 = self.data.loc[self.timesteps,'HIGH'] < (self.data.loc[self.timesteps,'LOW'] * 1.5) # average
    f5 = self.data.loc[self.timesteps,'VOLUME'] < (self.data.loc[self.timesteps,'LOW'] * 4) # average
    f6 = self.data.loc[self.timesteps,'OPEN'] < self.data.loc[self.timesteps,'CLOSE']
    f7 = self.data.loc[self.timesteps,'WEEKDAY'] > 4 # weekend
    #f8 = 4 < self.data.loc[self.timesteps,'MONTH'] < 8 # summer
    #f9 = 10 < self.data.loc[self.timesteps,'DAY'] < 20 # summer
    features = [f0,f1,f2,f3,f4,f5,f6,f7]
    return ''.join(['1' if f else '0' for f in features])

  def step(self, action):
    # take one timestep and check if we have more samples
    self.timesteps += 1
    if self.timesteps == len(self.data) - 1:
      terminal = True
    else:
      terminal = False
    
    # perform action
    if(action == 1):   # buy
      price = self.data.loc[self.timesteps,'OPEN'] * 1.002 # 0.2% transaction fee
      if(self.capital < price):
        tqdm.write('Insufficient balance')
      else:
        #self.satoshi += 1
        self.satoshi_queue[(self.timesteps - 1) % self.delay] += 1 # transaction delay
        self.capital -= price # immediate payment
    elif(action == 2): # sell
      if(self.satoshi < 1):
        tqdm.write('Empty wallet')
      else:
        price = self.data.loc[self.timesteps,'OPEN'] * 1.002 # 0.2% transaction fee
        self.satoshi -= 1 # immediate payment
        #self.capital += price
        self.capital_queue[(self.timesteps - 1) % self.delay] += price # transaction delay

    self.capital += self.capital_queue[self.timesteps % self.delay]
    self.satoshi += self.satoshi_queue[self.timesteps % self.delay]
    self.capital_queue[self.timesteps % self.delay] = 0
    self.satoshi_queue[self.timesteps % self.delay] = 0
    
    reward = (self.capital + np.sum(self.capital_queue)) - 0.01 + \
              self.data.loc[self.timesteps,'CLOSE'] * (self.satoshi + np.sum(self.satoshi_queue))
    state = self.state_maker()
    return state, reward, terminal


class DynaQ(object):
  def __init__(self, n_actions=3): # 0: hold, 1: buy, 2: sell
    self.n_actions = n_actions
    self.Model = {}
    self.Q = {}

    # Hyperparameters
    self.alpha = 1
    self.epsilon = 0.1
    self.gamma = 0.95

  def update_model(self, s, a, r, s_dash):
    if s in self.Model:
      if not type(self.Model[s]) == dict:
        self.Model[s] = {}
      self.Model[s][a] = (r, s_dash)
    else:
      self.Model[s] = {a: (r, s_dash)}

  def sample_model(self):
    s = np.random.choice(list(self.Model.keys()))
    a = np.random.choice(list(self.Model[s].keys()))
    r, s_dash = self.Model[s][a]
    return s, a, r, s_dash

  def update_q(self, s, a, r, s_dash):
    if s_dash not in self.Q:
      self.Q[s_dash] = np.zeros(self.n_actions)

    self.Q[s][a] += self.alpha * (r + self.gamma * self.Q[s_dash].max() - self.Q[s][a])

  def check_valid_action(self, s, a):
    if a == 1 and s[1] == '0': # buy and not enough balance
      return False
    elif a == 2 and s[0] == '0': # sell and empty wallet
      return False
    else:
      return True

  def sample_q(self, s):
    if s not in self.Q:
      self.Q[s] = np.zeros(self.n_actions)

    while True:
      if np.random.rand() < self.epsilon:
        a = np.random.choice(self.n_actions)
      else:
        a = np.random.choice(np.flatnonzero(self.Q[s] == self.Q[s].max()))
      
      if self.check_valid_action(s, a):
        return a

! git clone https://github.com/tqdm/tqdm

from tqdm.tqdm import tqdm

def learn(n_steps=6000, n_imaginations=50, delay=25, den=10, num=1):
  alg = DynaQ()
  env = Env(delay=delay)
  epsilon_schedule = [.5]*500 + [.1]* (n_steps - 500)
  alpha_schedule = num / np.arange(den, n_steps + den)

  s = env.reset()
  rewards = []
  sat = []
  cap = []
  for i in tqdm(range(n_steps)):
    alg.epsilon = epsilon_schedule[i]
    alg.alpha = alpha_schedule[i]
    a = alg.sample_q(s)
    s_dash, r, terminal = env.step(a)
    alg.update_q(s, a, r, s_dash)
    alg.update_model(s, a, r, s_dash)

    rewards.append(r)

    for j in range(n_imaginations):
      experience = alg.sample_model()
      alg.update_q(*experience)

    if terminal == True:
      s = env.reset()
    else:
      s = s_dash
  satoshi = env.satoshi
  print('Number of Satoshis %d worth of %f'% (satoshi, env.data.loc[env.timesteps,'CLOSE'] * satoshi))
  print('Capital of %f and in total %f'% (env.capital, env.data.loc[env.timesteps,'CLOSE'] * satoshi + env.capital) )
  for i in range(2**8):
    s = format(i, '08b')
    if s not in alg.Q:
      #print("%s is the first time happening" %s)
      continue
    else:
      foo = np.flatnonzero(alg.Q[s] == alg.Q[s].max())
      if foo.size == 0:
        print("%s is empty!?" %s)
      else:
        a = np.random.choice(foo)
        print((s,a), end=", ")
  return rewards, env.satoshi, (env.data.loc[env.timesteps,'CLOSE'] * env.satoshi + env.capital)

"""###Learning with Dyna-Q algorithm:"""

for i in range(10):
  r, s, c = learn(n_steps=172137, n_imaginations=100, delay=100)
  if i == 0:
    arr = r
    sat = s
    cap = c
    continue
  arr = [sum(x) for x in zip(arr, r)]
  sat += s
  cap += c

plt.plot([ar/10 for ar in arr])
_ = plt.title("Collected imaginary reward of Dyna-Q Algorithm averaged over 10 runs")

print('Satoshi of %f and capital of %f'% (sat/10, cap/10) )

"""###Learning with random policy:"""

for i in range(10):
  r, s, c = learn(n_steps=172137, n_imaginations=0, delay=100)
  if i == 0:
    arr = r
    sat = s
    cap = c
    continue
  arr = [sum(x) for x in zip(arr, r)]
  sat += s
  cap += c
plt.figure()
plt.plot([ar/10 for ar in arr])
_ = plt.title("Collected imaginary reward of random policy averaged over 10 runs")
plt.show()

print('Capital of %f and capital in total %f'% (sat/10, cap/10) )