Skip to content

Slicing a dataframe #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Slicing a dataframe #45

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6cb61d9
Slicing a dataframe
Nov 10, 2018
3d3c3a3
Update data_processor.py
Nov 10, 2018
a339110
importing
Nov 11, 2018
ccb858b
Update config.json
Nov 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 17 additions & 14 deletions core/data_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,13 @@ def get_test_data(self, seq_len, normalise):
Warning: batch method, not generative, make sure you have enough memory to
load data, otherwise reduce size of the training split.
'''
data_windows = []
data_x = []
data_y = []
for i in range(self.len_test - seq_len):
data_windows.append(self.data_test[i:i+seq_len])

data_windows = np.array(data_windows).astype(float)
data_windows = self.normalise_windows(data_windows, single_window=False) if normalise else data_windows

x = data_windows[:, :-1]
y = data_windows[:, -1, [0]]
return x,y
x, y = self._next_window(i, seq_len, normalise, train=False)
data_x.append(x)
data_y.append(y)
return np.array(data_x), np.array(data_y)

def get_train_data(self, seq_len, normalise):
'''
Expand All @@ -40,7 +37,7 @@ def get_train_data(self, seq_len, normalise):
data_x = []
data_y = []
for i in range(self.len_train - seq_len):
x, y = self._next_window(i, seq_len, normalise)
x, y = self._next_window(i, seq_len, normalise, train=True)
data_x.append(x)
data_y.append(y)
return np.array(data_x), np.array(data_y)
Expand All @@ -62,23 +59,29 @@ def generate_train_batch(self, seq_len, batch_size, normalise):
i += 1
yield np.array(x_batch), np.array(y_batch)

def _next_window(self, i, seq_len, normalise):
def _next_window(self, i, seq_len, normalise, train=True):
'''Generates the next data window from the given index location i'''
window = self.data_train[i:i+seq_len]

if train:
window = self.data_train[i:i+seq_len+1]
else:
window = self.data_test[i:i+seq_len+1]
window = self.normalise_windows(window, single_window=True)[0] if normalise else window
x = window[:-1]
y = window[-1, [0]]
return x, y

def normalise_windows(self, window_data, single_window=False):
'''Normalise window with a base value of zero'''
eps = 0.00001
normalised_data = []
window_data = [window_data] if single_window else window_data
for window in window_data:
normalised_window = []
for col_i in range(window.shape[1]):
normalised_col = [((float(p) / float(window[0, col_i])) - 1) for p in window[:, col_i]]

normalised_col = [((float(p) / (float(window[0, col_i]) + eps) ) - 1) for p in window[:, col_i]]
normalised_window.append(normalised_col)
normalised_window = np.array(normalised_window).T # reshape and transpose array back into original multidimensional format
normalised_data.append(normalised_window)
return np.array(normalised_data)
return np.array(normalised_data)