Merge pull request #52 from ArnovanHilten/dev

Dev: implemented network from .npz

Author: ArnovanHilten

GenNet_utils/Create_network.py

@@ -38,12 +38,11 @@ def example_network():
     return model, masks
 
 def layer_block(model, mask, i):
-    masks = []
     model = LocallyDirected1D(mask=mask, filters=1, input_shape=(mask.shape[0], 1),
                               name="LocallyDirected_" + str(i))(model)
     model = K.layers.Activation("tanh")(model)
     model = K.layers.BatchNormalization(center=False, scale=False)(model)
-    return model, masks
+    return model
 
 def create_network_from_csv(datapath, l1_value=0.01, regression=False):
     masks = []
@@ -60,12 +59,13 @@ def create_network_from_csv(datapath, l1_value=0.01, regression=False):
     model = K.layers.Reshape(input_shape=(inputsize,), target_shape=(inputsize, 1))(input_layer)
 
     for i in range(len(columns) - 1):
-        matrix_ones = np.ones(len(network_csv[[columns[i], columns[i + 1]]]), np.bool)
-        matrix_coord = (network_csv[columns[i]].values, network_csv[columns[i + 1]].values)
+        network_csv2 = network_csv.drop_duplicates(columns[i])
+        matrix_ones = np.ones(len(network_csv2[[columns[i], columns[i + 1]]]), np.bool)
+        matrix_coord = (network_csv2[columns[i]].values, network_csv2[columns[i + 1]].values)
         if i == 0:
-            matrixshape = (inputsize, network_csv[columns[i + 1]].max() + 1)
+            matrixshape = (inputsize, network_csv2[columns[i + 1]].max() + 1)
         else:
-            matrixshape = (network_csv[columns[i]].max() + 1, network_csv[columns[i + 1]].max() + 1)
+            matrixshape = (network_csv2[columns[i]].max() + 1, network_csv2[columns[i + 1]].max() + 1)
         mask = scipy.sparse.coo_matrix(((matrix_ones), matrix_coord), shape = matrixshape)
         masks.append(mask)
         model = layer_block(model, mask, i)
@@ -86,7 +86,6 @@ def create_network_from_csv(datapath, l1_value=0.01, regression=False):
     return model, masks
 
 def create_network_from_npz(datapath, l1_value=0.01, regression=False):
-    print('ToDO: test')
     masks = []
     mask_shapes_x = []
     mask_shapes_y = []
@@ -104,7 +103,9 @@ def create_network_from_npz(datapath, l1_value=0.01, regression=False):
     for i in range(len(masks)):  # sort all the masks in the correct order
         argsort_x = np.argsort(mask_shapes_x)[::-1]
         argsort_y = np.argsort(mask_shapes_y)[::-1]
-
+        
+        mask_shapes_x = np.array(mask_shapes_x)
+        mask_shapes_y = np.array(mask_shapes_y)
         assert all(argsort_x == argsort_y) # check that both dimensions have the same order
 
         masks  = [masks[i] for i in argsort_y] # sort masks
@@ -123,7 +124,7 @@ def create_network_from_npz(datapath, l1_value=0.01, regression=False):
     input_layer = K.Input((inputsize,), name='input_layer')
     model = K.layers.Reshape(input_shape=(inputsize,), target_shape=(inputsize, 1))(input_layer)
 
-    for i in range(len(masks) - 1):
+    for i in range(len(masks)):
         mask = masks[i]
         model = layer_block(model, mask, i)
 
@@ -148,10 +149,9 @@ def create_network_from_npz(datapath, l1_value=0.01, regression=False):
 
 
 def lasso(inputsize, l1_value):
-    masks = []
     inputs = K.Input((inputsize,), name='inputs')
     x1 = K.layers.BatchNormalization(center=False, scale=False, name="inter_out")(inputs)
     x1 = K.layers.Dense(units=1, kernel_regularizer=K.regularizers.l1(l1_value))(x1)
     x1 = K.layers.Activation("sigmoid")(x1)
     model = K.Model(inputs=inputs, outputs=x1)
-    return model, masks
+    return model

GenNet_utils/Dataloader.py

@@ -1,6 +1,6 @@
 import os
 import sys
-
+import glob
 sys.path.insert(1, os.path.dirname(os.getcwd()) + "/GenNet_utils/")
 import matplotlib
 import numpy as np
@@ -23,7 +23,7 @@ def check_data(datapath, mode):
         print("genotype.h5 is missing")
     if os.path.exists(datapath + 'topology.csv'):
         network_structure = True
-    elif os.path.exists(datapath + '*.npz'):
+    elif len(glob.glob(datapath + '*.npz'))>0:
         network_structure = True
     else:
         print("topology.csv and *.npz are missing")

GenNet_utils/LocallyDirectedConnected.py

@@ -146,7 +146,7 @@ def build(self, input_shape):
             raise ValueError('Axis 2 of input should be fully-defined. '
                              'Found shape:', input_shape)
         self.output_length = self.mask.shape[1]
-        print("output length is " + str(self.output_length))
+#         print("output length is " + str(self.output_length))
         if self.data_format == 'channels_first':
             self.kernel_shape = (input_dim, input_length,
                                  self.filters, self.output_length)
@@ -267,7 +267,7 @@ def get_locallyDirected1D_mask(mask, kernel, data_format,
 
     ndims = int(mask.ndim / 2)
     indices = np.mat([mask.row, mask.col]).transpose()
-    print(mask.shape)
+#     print(mask.shape)
     mask = tf.SparseTensor(indices, kernel, [mask.shape[0], mask.shape[1]])
 
     if data_format == 'channels_first':
@@ -429,7 +429,7 @@ def make_2d(tensor, split_dim):
     Tensor of shape
     `(d0 * ... * d(split_dim-1), d(split_dim) * ... * d(N-1))`.
   """
-    print(tensor.shape)
+#     print(tensor.shape)
     shape = K.array_ops.shape(tensor)
     in_dims = shape[:split_dim]
     out_dims = shape[split_dim:]

GenNet_utils/LocallyDirectedConnected_tf2.py

@@ -252,9 +252,9 @@ def local_conv_matmul_sparse(inputs, mask, kernel, kernel_idx, output_length, fi
     output_shape = (mask.shape[1], mask.shape[0])
     inputs_flat = K.reshape(inputs, (K.shape(inputs)[0], -1))
 
-    print("kernel_idx", len(kernel_idx))
-    print("inputs", K.shape(inputs_flat))
-    print("kernel", K.shape(kernel))
+#     print("kernel_idx", len(kernel_idx))
+#     print("inputs", K.shape(inputs_flat))
+#     print("kernel", K.shape(kernel))
 
     output_flat = K.sparse_ops.sparse_tensor_dense_mat_mul(
         kernel_idx, kernel, (mask.shape[1], mask.shape[0]), inputs_flat, adjoint_b=True)

GenNet_utils/Train_network.py

@@ -130,8 +130,10 @@ def train_classification(args):
         f.write('\n Confusion matrix ')
         f.write(str(confusionmatrix_test))
 
-    importance_csv = create_importance_csv(datapath, model, masks)
-    importance_csv.to_csv(resultpath + "connection_weights.csv")
+    if os.path.exists(datapath + "/topology.csv"):
+        importance_csv = create_importance_csv(datapath, model, masks)
+        importance_csv.to_csv(resultpath + "connection_weights.csv")
+
 
 
 def train_regression(args):
@@ -231,5 +233,7 @@ def train_regression(args):
         f.write('\n Explained variance = ' + str(explained_variance_val))
         # f.write('\n Maximum error = ' + str(maximum_error_test))
         f.write('\n R2 = ' + str(r2_test))
-    importance_csv = create_importance_csv(datapath, model, masks)
-    importance_csv.to_csv(resultpath + "connection_weights.csv")
+        
+    if os.path.exists(datapath + "/topology.csv"):
+        importance_csv = create_importance_csv(datapath, model, masks)
+        importance_csv.to_csv(resultpath + "connection_weights.csv")

README.md

@@ -57,7 +57,8 @@ pip3 install -r requirements_GenNet.txt
 
 ```
 *GenNet is ready to use!*
-Navigate to the GenNet foler and use the following command to run the example:
+
+Navigate to the GenNet folder and use the following command to run the example:
 ```
 python GenNet.py train ./examples/example_classification/ 1
 ```
@@ -128,9 +129,9 @@ python GenNet.py topology --help
 
 #### GenNet output
 
-After training your network it saved together with its results. Results include a text file with the performance, a .CSV file with all the connections and their weights, a .h5 with the best weights on the validtion set and a plot of the training and validation loss. 
+After training your network it saved together with its results. Results include a text file with the performance, a .CSV file with all the connections and their weights, a .h5 with the best weights on the validation set and a plot of the training and validation loss. 
 
-The .CSV file with the weights can be used to create your own plot but `python GenNet.py plot` also has standard plots availabe:
+The .CSV file with the weights can be used to create your own plot but `python GenNet.py plot` also has standard plots available:
 
 
 ##### Manhattan plot
@@ -142,7 +143,7 @@ The .CSV file with the weights can be used to create your own plot but `python G
 
 ### Jupyter notebook
 
-The orignal jupyter notebooks can be found in the jupyter notebook folder. Navigate to the jupyter notebook folder and start with `jupyter notebook`
+The original jupyter notebooks can be found in the jupyter notebook folder. Navigate to the jupyter notebook folder and start with `jupyter notebook`. The notebooks are not updated but can be a useful source to understand the main code and/or to create .npz masks (to define connections between layers).
 
 ### More
 

Submit_slurm_GenNet_train.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH --mem=10G
+#SBATCH --ntasks=6
+#SBATCH -p express 
+#SBATCH --gres=gpu:1
+#SBATCH -t 01:00:00
+#SBATCH -o /trinity/home/avanhilten/repositories/GenNet/GenNet_utils/SLURM_logs/slurm_logs/out_%j.log
+#SBATCH -e /trinity/home/avanhilten/repositories/GenNet/GenNet_utils/SLURM_logs/error_%j.log
+
+# Load the modules
+
+module purge
+module load Python/3.7.4-GCCcore-8.3.0
+module load libs/cuda/10.1.243
+module load libs/cudnn/7.6.5.32-CUDA-10.1.243
+module load TensorFlow/2.2.0-fosscuda-2019b-Python-3.7.4
+
+source $HOME/env_GenNet_dev/bin/activate
+
+cd /trinity/home/avanhilten/repositories/GenNet/
+
+python GenNet.py train ./examples/example_classification/ 111