training_loop.py

import numpy as np
import urllib.request

# Load Shakespeare dataset
url = "https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
text = urllib.request.urlopen(url).read().decode('utf-8')
print("Dataset length:", len(text))

chars = list(set(text))
vocab_size = len(chars)
char_to_ix = { ch:i for i,ch in enumerate(chars) }
ix_to_char = { i:ch for i,ch in enumerate(chars) }
print("Unique chars:", vocab_size)


# Defining RNN class
class RNN:
    def __init__(self, input_size, hidden_size, output_size):
        self.Wxh = np.random.randn(hidden_size, input_size) * 0.01
        self.Whh = np.random.randn(hidden_size, hidden_size) * 0.01
        self.Why = np.random.randn(output_size, hidden_size) * 0.01
        self.bh = np.zeros((hidden_size, 1))
        self.by = np.zeros((output_size, 1))
        self.h_prev = np.zeros((hidden_size, 1))
        
    def softmax(self, x):
        e_x = np.exp(x - np.max(x))
        return e_x / e_x.sum(axis=0)
    
    def forward(self, inputs):
        xs, hs, ys, ps = {}, {}, {}, {}
        hs[-1] = np.copy(self.h_prev)
        for t in range(len(inputs)):
            xs[t] = np.zeros((vocab_size, 1))
            xs[t][inputs[t]] = 1
            hs[t] = np.tanh(np.dot(self.Wxh, xs[t]) + np.dot(self.Whh, hs[t-1]) + self.bh)
            ys[t] = np.dot(self.Why, hs[t]) + self.by
            ps[t] = self.softmax(ys[t])
        self.h_prev = hs[len(inputs)-1]
        return xs, hs, ys, ps
        
    def backward(self, inputs, targets, xs, hs, ys, ps):
        dWxh, dWhh, dWhy = np.zeros_like(self.Wxh), np.zeros_like(self.Whh), np.zeros_like(self.Why)
        dbh, dby = np.zeros_like(self.bh), np.zeros_like(self.by)
        dh_next = np.zeros_like(hs[0])

        for t in reversed(range(len(inputs))):
            dy = np.copy(ps[t])
            dy[targets[t]] -= 1
            dWhy += np.dot(dy, hs[t].T)
            dby += dy
            dh = np.dot(self.Why.T, dy) + dh_next
            dhraw = (1 - hs[t] * hs[t]) * dh
            dbh += dhraw
            dWxh += np.dot(dhraw, xs[t].T)
            dWhh += np.dot(dhraw, hs[t-1].T)
            dh_next = np.dot(self.Whh.T, dhraw)

        for dparam in [dWxh, dWhh, dWhy, dbh, dby]:
            np.clip(dparam, -5, 5, out=dparam)
        return dWxh, dWhh, dWhy, dbh, dby

    def update_parameters(self, dWxh, dWhh, dWhy, dbh, dby, learning_rate):
        self.Wxh -= learning_rate * dWxh
        self.Whh -= learning_rate * dWhh
        self.Why -= learning_rate * dWhy
        self.bh  -= learning_rate * dbh
        self.by  -= learning_rate * dby


# Sampling function (from a trained model)
def sample(rnn, seed_ix, n):
    x = np.zeros((vocab_size, 1))
    x[seed_ix] = 1
    h = rnn.h_prev
    ixes = []
    for t in range(n):
        h = np.tanh(np.dot(rnn.Wxh, x) + np.dot(rnn.Whh, h) + rnn.bh)
        y = np.dot(rnn.Why, h) + rnn.by
        p = rnn.softmax(y)
        ix = np.random.choice(range(vocab_size), p=p.ravel())
        x = np.zeros((vocab_size, 1))
        x[ix] = 1
        ixes.append(ix)
    return ixes


# Training loop
hidden_size = 100
seq_length = 25
learning_rate = 1e-1

rnn = RNN(vocab_size, hidden_size, vocab_size)

n, p = 0, 0
smooth_loss = -np.log(1.0/vocab_size)*seq_length
for epoch in range(10000):
    if p + seq_length + 1 >= len(text) or n == 0:
        rnn.h_prev = np.zeros((hidden_size,1))
        p = 0

    inputs = [char_to_ix[ch] for ch in text[p:p+seq_length]]
    targets = [char_to_ix[ch] for ch in text[p+1:p+seq_length+1]]

    xs, hs, ys, ps = rnn.forward(inputs)
    loss = sum(-np.log(ps[t][targets[t],0]) for t in range(seq_length))
    smooth_loss = smooth_loss * 0.999 + loss * 0.001

    dWxh, dWhh, dWhy, dbh, dby = rnn.backward(inputs, targets, xs, hs, ys, ps)
    rnn.update_parameters(dWxh, dWhh, dWhy, dbh, dby, learning_rate)

    if n % 1000 == 0:
        print(f"Iteration {n}, loss: {smooth_loss:.2f}")
        sample_ix = sample(rnn, inputs[0], 200)
        txt = ''.join(ix_to_char[ix] for ix in sample_ix)
        print(f"----\n{txt}\n----")

    p += seq_length
    n += 1