Merge pull request #78 from ArnovanHilten/dev

Add automatic testing

Author: ArnovanHilten

.github/workflows/tests.yml

@@ -0,0 +1,25 @@
+name: PyTests
+on:
+ - push
+ - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6","3.7", "3.8"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f  requirements_GenNet.txt ]; then pip install -r  requirements_GenNet.txt; fi
+      - name: Test with pytest
+        run: |
+          pytest

GenNet_utils/LocallyDirectedConnected_tf2.py

@@ -158,12 +158,12 @@ def build(self, input_shape):
             self.kernel_shape = (input_length, input_dim,
                                  self.output_length, self.filters)
 
-        self.kernel = self.add_weight(shape=(len(self.mask.data),),  # sum of all nonzero values in mask sum(sum(mask))
+        self.kernel = self.add_weight(shape=(len(self.mask.data)*self.filters,),  # sum of all nonzero values in mask sum(sum(mask))
                                       initializer=self.kernel_initializer,
                                       name='kernel',
                                       regularizer=self.kernel_regularizer,
                                       constraint=self.kernel_constraint)
-        self.kernel_idx = sorted(self.get_idx(self.mask))
+        self.kernel_idx = self.get_idx(self.mask)
 
         if self.use_bias:
             self.bias = self.add_weight(
@@ -182,8 +182,7 @@ def build(self, input_shape):
         self.built = True
 
     def call(self, inputs):
-        output = self.local_conv_matmul_sparse(inputs, self.mask, self.kernel, self.kernel_idx, self.output_length,
-                                          self.filters)
+        output = self.local_conv_matmul_sparse(inputs, self.mask, self.kernel, self.kernel_idx, self.output_length)
         if self.use_bias:
             output = K.bias_add(output, self.bias, data_format=self.data_format)
 
@@ -224,7 +223,7 @@ def get_config(self):
         return dict(list(base_config.items()) + list(config.items()))
 
 
-    def local_conv_matmul_sparse(self, inputs, mask, kernel, kernel_idx, output_length, filters):
+    def local_conv_matmul_sparse(self, inputs, mask, kernel, kernel_idx, output_length):
         """Apply N-D convolution with un-shared weights using a single matmul call.
 
       Arguments:
@@ -256,19 +255,23 @@ def local_conv_matmul_sparse(self, inputs, mask, kernel, kernel_idx, output_leng
         inputs_flat = K.reshape(inputs, (K.shape(inputs)[0], -1))
 
         output_flat = K.sparse_ops.sparse_tensor_dense_mat_mul(
-            kernel_idx, kernel, (mask.shape[1], mask.shape[0]), inputs_flat, adjoint_b=True)
+            kernel_idx, kernel, (mask.shape[1] * self.filters, mask.shape[0]), inputs_flat, adjoint_b=True)
 
         output_flat_transpose = K.transpose(output_flat)
-        output_reshaped = K.reshape(output_flat_transpose, [-1, output_length, filters])
+        output_reshaped = K.reshape(output_flat_transpose, [-1, output_length, self.filters]) # gene dimension extension shaped back
         return output_reshaped
 
 
     def get_idx(self, mask):
         """"returns the transposed coordinates in tuple form:
          [(mask.col[0], mask,row[0])...[mask.col[n], mask.row[n])]"""
         coor_list = []
-        for i, j in zip(mask.col, mask.row):
-            coor_list.append((i, j))
-
+        offset = 0
+        for fr in range(self.filters):   # for each filter the gene/output dimension is expanded.
+            filter_list=[]
+            for i, j in zip(mask.col, mask.row):
+                filter_list.append((i + offset, j))
+            coor_list = coor_list  + sorted(filter_list) # we sort per filter
+            offset += mask.shape[1]
         return coor_list
 

README.md

@@ -1,4 +1,5 @@
 [![DOI](https://zenodo.org/badge/240289809.svg)](https://zenodo.org/badge/latestdoi/240289809)
+![Build](https://github.com/ArnovanHilten/GenNet/actions/workflows/tests.yml/badge.svg)
 
 # **GenNet**
 **Framework for Interpretable Neural Networks for Genetics**

requirements_GenNet.txt

@@ -15,6 +15,9 @@ sklearn>=0.0
 tables>=3.6.1
 tqdm>=4.49.0
 zipp>=3.1.0
+pytest>=6.2.5
+protobuf>=3.11,<=3.20.2
 psutil
 kaleido
 tensorflow==2.2.0
+bitarray

tests/test_GenNet.py

@@ -6,17 +6,16 @@
 from GenNet_utils.Create_plots import  sunburst_plot, plot_layer_weight, manhattan_relative_importance
 from GenNet_utils.Utility_functions import get_paths
 
-# import unittest
 # TODO: add test without covariates
 # TODO add test with covariates for regression + classification
 # TODO add test with multiple genotype files.
 # test randomnesss after .. epoch shuffles.
-
+# ToDO add test for each file.
 
 def get_GenNet_path():
     return str(dirname(dirname(abspath(__file__)))) + "/"
 
-def remove_old_test(ID):
+def remove_old(ID):
     GenNet_path  =  get_GenNet_path()
     resultpath = GenNet_path +  "results/GenNet_experiment_" + str(ID)  + '_/'
     try:
@@ -71,31 +70,44 @@ def test_convert(self):
             " -o {}/examples/A_to_Z/processed_data/"
             "/  -study_name GenNet_simulation -step all".format(GenNet_path, GenNet_path, GenNet_path) )
         assert test1 == 0
-        
-#         !python GenNet.py topology -type create_annovar_input -path ./examples/A_to_Z/processed_data/ -study_name GenNet_simulation -out examples/A_to_Z/processed_data/
-
-# !python GenNet.py topology -type create_gene_network -path examples/A_to_Z/processed_data/ -out examples/A_to_Z/processed_data/ -study_name GenNet_simulation
-
-
-# mkdir examples/A_to_Z/new_run_folder/
-# mv examples/A_to_Z/processed_data/SNP_gene_mask.npz  examples/A_to_Z/new_run_folder/ # or topology.csv
-# mv examples/A_to_Z/processed_data/genotype.h5  examples/A_to_Z/new_run_folder/
-# cp examples/A_to_Z/run_folder/subjects.csv  examples/A_to_Z/new_run_folder/ 
-
+ 
+    def test_topology_annovar_input(self):
+        GenNet_path = get_GenNet_path()
+        test1 = os.system(
+            "python {}/GenNet.py topology -type create_annovar_input -path {}//examples/A_to_Z/processed_data/ -study_name GenNet_simulation -out {}/examples/A_to_Z/processed_data/".format(GenNet_path, GenNet_path, GenNet_path) )
+        assert test1 == 0
 
-# !python GenNet.py train -path examples/A_to_Z/new_run_folder/ -ID 100001 -epochs 50
+    def test_topology_create_gene_network(self):
+        GenNet_path = get_GenNet_path()
+        test1 = os.system(
+            "python {}/GenNet.py topology -type create_gene_network -path {}/examples/A_to_Z/processed_data/ -out {}/examples/A_to_Z/processed_data/ -study_name GenNet_simulation".format(GenNet_path, GenNet_path, GenNet_path) )
+        assert test1 == 0
 
-# !python GenNet.py plot -ID 100001 -type manhattan_relative_importance 
+    def test_train_run(self): 
+        GenNet_path = get_GenNet_path()
+        os.system('mkdir {}/examples/A_to_Z/new_run_folder/'.format(GenNet_path))
+        os.system('cp {}/examples/A_to_Z/processed_data/topology.csv  {}/examples/A_to_Z/new_run_folder/'.format(GenNet_path, GenNet_path))
+        os.system('cp {}/examples/A_to_Z/processed_data/genotype.h5  {}/examples/A_to_Z/new_run_folder/'.format(GenNet_path, GenNet_path))
+        os.system('cp {}/examples/A_to_Z/run_folder/subjects.csv  {}/examples/A_to_Z/new_run_folder/'.format(GenNet_path, GenNet_path))
+        test1 = os.system('python {}/GenNet.py train -path {}/examples/A_to_Z/new_run_folder/ -ID 999999997 -epochs 5'.format(GenNet_path, GenNet_path))
+        assert test1 == 0
+        
+    def test_manhattan_plot(self):
+        GenNet_path = get_GenNet_path()
+        test1 = os.system(
+            "python {}/GenNet.py plot -ID 999999997 -type manhattan_relative_importance".format(GenNet_path) )
+        assert test1 == 0        
 
+        
 class TestTrain():
     def test_train_classification(self): 
-        remove_old_test(999999999)
+        remove_old(999999999)
         GenNet_path = get_GenNet_path()
         value = os.system('python {}/GenNet.py train -path {}/examples/example_classification/ -ID 999999999 -epochs 2'.format(GenNet_path, GenNet_path) )
         assert value == 0
 
     def test_train_regression(self):
-        remove_old_test(999999998)
+        remove_old(999999998)
         GenNet_path = get_GenNet_path()
         value = os.system('python {}/GenNet.py train -path {}/examples/example_regression/ -ID 999999998 -problem_type regression -epochs 2'.format(GenNet_path, GenNet_path) )
         assert value == 0        
@@ -127,8 +139,5 @@ def test_plot_layer_weight_2(self, ID):
         plot_layer_weight(resultpath, importance_csv, layer=1)
 
 
-# @pytest.mark.parametrize("ID", [999999999, 999999998]) # test both regression and classification      
-# class CheckAllFiles():
-#     def trainin_files(self, ID):