Skip to content

Commit 33408bd

Browse files
ArnovanHiltenArnovanHilten
authored
Merge pull request #21 from ArnovanHilten/dev
Dev
2 parents e6dc025 + d4adea5 commit 33408bd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+140
-165
lines changed

‎GenNet.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
import sys
21
import os
2+
import sys
33
import warnings
44

55
warnings.filterwarnings('ignore')
66
import argparse
77

8-
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/utils/")
9-
from utils.Create_plots import plot
10-
from utils.Train_network import train_classification, train_regression
11-
from utils.Convert import convert
8+
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/GenNet_utils/")
9+
from GenNet_utils.Create_plots import plot
10+
from GenNet_utils.Train_network import train_classification, train_regression
11+
from GenNet_utils.Convert import convert
1212

1313

1414
def main(args):

‎utils/Convert.py ‎GenNet_utils/Convert.py

+10-8
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,21 @@
1-
import sys
1+
import glob
22
import os
3+
import sys
4+
35
import h5py
4-
import glob
5-
import tables
6-
import tqdm
76
import numpy as np
87
import pandas as pd
9-
from utils.hase.config import basedir, CONVERTER_SPLIT_SIZE, PYTHON_PATH
8+
import tables
9+
import tqdm
10+
11+
from GenNet_utils.hase.config import basedir, CONVERTER_SPLIT_SIZE, PYTHON_PATH
1012

1113
os.environ['HASEDIR'] = basedir
1214
if PYTHON_PATH is not None:
1315
for i in PYTHON_PATH: sys.path.insert(0, i)
14-
from utils.hase.hdgwas.tools import Timer, check_converter
15-
from utils.hase.hdgwas.converter import GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
16-
from utils.hase.hdgwas.data import Reader
16+
from GenNet_utils.hase.hdgwas.tools import Timer, check_converter
17+
from GenNet_utils.hase.hdgwas.converter import GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
18+
from GenNet_utils.hase.hdgwas.data import Reader
1719

1820

1921
def hase_convert(args):

‎utils/Create_network.py ‎GenNet_utils/Create_network.py

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import os
22
import sys
3+
34
import numpy as np
45
import pandas as pd
56

6-
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/utils/")
7+
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/GenNet_utils/")
78
import matplotlib
89

910
matplotlib.use('agg')
@@ -14,12 +15,12 @@
1415
tf.keras.backend.set_epsilon(0.0000001)
1516
tf_version = tf.__version__ # ToDo use packaging.version
1617
if tf_version <= '1.13.1':
17-
from utils.LocallyDirectedConnected import LocallyDirected1D
18+
from GenNet_utils.LocallyDirectedConnected import LocallyDirected1D
1819
elif tf_version >= '2.0':
19-
from utils.LocallyDirectedConnected_tf2 import LocallyDirected1D
20+
from GenNet_utils.LocallyDirectedConnected_tf2 import LocallyDirected1D
2021
else:
2122
print("unexpected tensorflow version")
22-
from utils.LocallyDirectedConnected_tf2 import LocallyDirected1D
23+
from GenNet_utils.LocallyDirectedConnected_tf2 import LocallyDirected1D
2324

2425

2526
def create_network_from_csv(datapath, l1_value=0.01, regression=False):
@@ -57,7 +58,7 @@ def layer_block(model, mask, i):
5758
output_layer = K.layers.Dense(units=1, name="output_layer",
5859
kernel_regularizer=tf.keras.regularizers.l1(l=l1_value))(model)
5960
if regression:
60-
output_layer = K.layers.Activation("relu")(output_layer)
61+
output_layer = K.layers.Activation("linear")(output_layer)
6162
else:
6263
output_layer = K.layers.Activation("sigmoid")(output_layer)
6364

‎utils/Create_plots.py ‎GenNet_utils/Create_plots.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import sys
2-
import os
3-
import numpy as np
4-
import pandas as pd
5-
import matplotlib
2+
63
import matplotlib.pyplot as plt
74
import networkx as nx
8-
from utils.utils import query_yes_no, get_paths
5+
import numpy as np
6+
import pandas as pd
97
import seaborn as sns
108

9+
from GenNet_utils.Utility_functions import query_yes_no, get_paths
10+
1111

1212
def plot(args):
1313
folder, resultpath = get_paths(args.ID)

‎utils/Dataloader.py ‎GenNet_utils/Dataloader.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
import sys
21
import os
2+
import sys
33

4-
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/utils/")
4+
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/GenNet_utils/")
55
import matplotlib
66
import numpy as np
77
import pandas as pd

‎utils/LocallyDirectedConnected.py ‎GenNet_utils/LocallyDirectedConnected.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
from __future__ import division
2424
from __future__ import print_function
2525

26+
import numpy as np
27+
import tensorflow as tf
2628
from tensorflow.python.keras import activations
2729
from tensorflow.python.keras import backend as K
2830
from tensorflow.python.keras import constraints
@@ -33,8 +35,6 @@
3335
from tensorflow.python.keras.utils import conv_utils
3436
from tensorflow.python.keras.utils import tf_utils
3537
from tensorflow.python.util.tf_export import tf_export
36-
import tensorflow as tf
37-
import numpy as np
3838

3939

4040
@tf_export('keras.layers.LocallyDirected1D')

‎utils/LocallyDirectedConnected_tf2.py ‎GenNet_utils/LocallyDirectedConnected_tf2.py

File renamed without changes.

‎utils/Train_network.py ‎GenNet_utils/Train_network.py

+8-7
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,20 @@
1-
import sys
21
import os
3-
import matplotlib
2+
import sys
43
import warnings
54

5+
import matplotlib
6+
67
warnings.filterwarnings('ignore')
78
matplotlib.use('agg')
8-
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/utils/")
9+
sys.path.insert(1, os.path.dirname(os.getcwd()) + "/GenNet_utils/")
910
import tensorflow as tf
1011
import tensorflow.keras as K
1112

1213
tf.keras.backend.set_epsilon(0.0000001)
13-
from utils.Dataloader import *
14-
from utils.utils import *
15-
from utils.Create_network import *
16-
from utils.Create_plots import *
14+
from GenNet_utils.Dataloader import *
15+
from GenNet_utils.Utility_functions import *
16+
from GenNet_utils.Create_network import *
17+
from GenNet_utils.Create_plots import *
1718

1819

1920
def weighted_binary_crossentropy(y_true, y_pred):

‎utils/utils.py ‎GenNet_utils/Utility_functions.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
1-
import pandas as pd
2-
import tensorflow as tf
31
import sys
4-
import tensorflow.keras as K
5-
import sklearn.metrics as skm
2+
63
import matplotlib.pyplot as plt
74
import numpy as np
5+
import pandas as pd
6+
import sklearn.metrics as skm
7+
import tensorflow as tf
8+
import tensorflow.keras as K
89

910
tf.keras.backend.set_epsilon(0.0000001)
1011
import os

‎utils/hase/hdgwas/__init__.py ‎GenNet_utils/__init__.py

File renamed without changes.

‎utils/hase/.gitignore ‎GenNet_utils/hase/.gitignore

File renamed without changes.

‎utils/hase/LICENSE ‎GenNet_utils/hase/LICENSE

File renamed without changes.

‎utils/hase/MANIFEST.in ‎GenNet_utils/hase/MANIFEST.in

File renamed without changes.

‎utils/hase/README.md ‎GenNet_utils/hase/README.md

File renamed without changes.

‎utils/hase/config.py ‎GenNet_utils/hase/config.py

File renamed without changes.

‎utils/hase/data/reference_file_link.txt ‎GenNet_utils/hase/data/reference_file_link.txt

File renamed without changes.

‎utils/hase/hase.py ‎GenNet_utils/hase/hase.py

+12-12
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,26 @@
1-
import sys
21
import os
2+
import sys
3+
34
import numpy as np
4-
from utils.hase.config import MAPPER_CHUNK_SIZE, basedir, CONVERTER_SPLIT_SIZE, PYTHON_PATH
5+
6+
from GenNet_utils.hase.config import MAPPER_CHUNK_SIZE, basedir, CONVERTER_SPLIT_SIZE, PYTHON_PATH
57

68
os.environ['HASEDIR'] = basedir
79
if PYTHON_PATH is not None:
810
for i in PYTHON_PATH: sys.path.insert(0, i)
9-
import h5py
10-
import tables
11-
from utils.hase.hdgwas.tools import Timer, Checker, study_indexes, Mapper, HaseAnalyser, merge_genotype, Reference, timing, \
11+
from GenNet_utils.hase.hdgwas.tools import Timer, Checker, study_indexes, Mapper, HaseAnalyser, merge_genotype, \
1212
check_np, check_converter
13-
from utils.hase.hdgwas.converter import GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
14-
from utils.hase.hdgwas.data import Reader, MetaParData, MetaPhenotype
15-
from utils.hase.hdgwas.fake import Encoder
16-
from utils.hase.hdgwas.hdregression import HASE, A_covariates, A_tests, B_covariates, C_matrix, A_inverse, B4
13+
from GenNet_utils.hase.hdgwas.converter import GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
14+
from GenNet_utils.hase.hdgwas.data import Reader, MetaParData, MetaPhenotype
15+
from GenNet_utils.hase.hdgwas.fake import Encoder
16+
from GenNet_utils.hase.hdgwas.hdregression import HASE, A_inverse, B4
1717
import argparse
1818
import gc
19-
from utils.hase.hdgwas.pard import partial_derivatives
20-
from utils.hase.hdgwas.regression import haseregression
19+
from GenNet_utils.hase.hdgwas.pard import partial_derivatives
20+
from GenNet_utils.hase.hdgwas.regression import haseregression
2121
import pandas as pd
2222
import time
23-
from utils.hase.hdgwas.protocol import Protocol
23+
from GenNet_utils.hase.hdgwas.protocol import Protocol
2424

2525
__version__ = '1.1.0'
2626

‎utils/hase/hase/__init__.py ‎GenNet_utils/hase/hase/__init__.py

+1-14
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,4 @@
1-
import sys
2-
import os
3-
import utils.hase.hdgwas.data
4-
from utils.hase.hdgwas.tools import Timer, Checker, study_indexes, Mapper, HaseAnalyser, merge_genotype, Reference, timing, \
5-
check_np
6-
from utils.hase.hdgwas.converter import GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
7-
from utils.hase.hdgwas.data import Reader, MetaParData, MetaPhenotype
8-
from utils.hase.hdgwas.fake import Encoder
9-
from utils.hase.hdgwas.hdregression import A_covariates, A_tests, B_covariates, C_matrix, A_inverse, B4
10-
from utils.hase.hdgwas.pard import partial_derivatives
11-
from utils.hase.hdgwas.regression import haseregression
12-
import time
13-
from utils.hase.hdgwas.protocol import Protocol
14-
from utils.hase.config import MAPPER_CHUNK_SIZE, basedir, CONVERTER_SPLIT_SIZE, PYTHON_PATH
1+
from GenNet_utils.hase.hdgwas.data import Reader
152

163

174
class HASE:

‎utils/hase/tools/__init__.py ‎GenNet_utils/hase/hdgwas/__init__.py

File renamed without changes.

‎utils/hase/hdgwas/converter.py ‎GenNet_utils/hase/hdgwas/converter.py

+9-7
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
1-
import os
2-
import tables
3-
import numpy as np
41
import gc
5-
import subprocess
6-
from utils.hase.hdgwas.tools import Timer
7-
import pandas as pd
8-
from utils.hase.hdgwas.data import MINIMACHDF5Folder
2+
import os
93
import shutil
104
import signal
5+
import subprocess
6+
7+
import numpy as np
8+
import pandas as pd
9+
import tables
10+
11+
from GenNet_utils.hase.hdgwas.data import MINIMACHDF5Folder
12+
from GenNet_utils.hase.hdgwas.tools import Timer
1113

1214

1315
class Genotype(object):

‎utils/hase/hdgwas/data.py ‎GenNet_utils/hase/hdgwas/data.py

+12-10
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
1-
import h5py
2-
import numpy as np
3-
import pandas as pd
4-
import os
51
import gc
6-
from numpy import genfromtxt
7-
import bitarray as ba
8-
from utils.hase.hdgwas.tools import Mapper
92
import glob
3+
import os
104
import shutil
115
from collections import OrderedDict
126

7+
import bitarray as ba
8+
import h5py
9+
import numpy as np
10+
import pandas as pd
11+
from numpy import genfromtxt
12+
13+
from GenNet_utils.hase.hdgwas.tools import Mapper
14+
1315

1416
class MINIMACPool(object):
1517

@@ -34,7 +36,7 @@ def get_chunk(self, indices=None):
3436
if indices is None:
3537
if self.finish != self.max_index:
3638
n = list(range(self.finish, self.finish + self.chunk_size)) if (
37-
self.finish + self.chunk_size) < self.max_index else list(
39+
self.finish + self.chunk_size) < self.max_index else list(
3840
range(self.finish, self.max_index))
3941
n = np.array(n)
4042
n_s = np.argsort(n)
@@ -274,7 +276,7 @@ def get_next(self, index=None, select_columns=None):
274276
else:
275277
start = self.processed
276278
finish = self.processed + self.chunk_size if (self.processed + self.chunk_size) <= self.shape[1] else \
277-
self.shape[1]
279+
self.shape[1]
278280
self.processed = finish
279281
if not isinstance(index, type(None)):
280282
self.start = start
@@ -413,7 +415,7 @@ def get(self):
413415
else:
414416
start = self.processed
415417
finish = self.processed + self.chunk_size if (
416-
self.processed + self.chunk_size) <= self.n_phenotypes else self.n_phenotypes
418+
self.processed + self.chunk_size) <= self.n_phenotypes else self.n_phenotypes
417419
self.processed = finish
418420
phenotype = np.zeros(
419421
(np.sum([len(self.pool[i].folder._data.id) for i in self.pool]), len(list(range(start, finish)))))

‎utils/hase/hdgwas/fake.py ‎GenNet_utils/hase/hdgwas/fake.py

+6-5
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import numpy as np
2-
import os
3-
import tables
41
import gc
2+
import os
3+
4+
import numpy as np
55
import pandas as pd
6+
import tables
67

78

89
class Encoder(object):
@@ -29,8 +30,8 @@ def __init__(self, out):
2930

3031
except:
3132
print((
32-
'Directories "encode_genotype","encode_phenotype","encode_individuals" are already exist in {}...'.format(
33-
self.out)))
33+
'Directories "encode_genotype","encode_phenotype","encode_individuals" are already exist in {}...'.format(
34+
self.out)))
3435

3536
def matrix(self, N, save=False):
3637

‎utils/hase/hdgwas/hash.py ‎GenNet_utils/hase/hdgwas/hash.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
import numpy as np
2-
import pickle as pickle
31
import os
2+
import pickle as pickle
3+
4+
import numpy as np
45

56

67
class HashTablePOS(object):

‎utils/hase/hdgwas/hdregression.py ‎GenNet_utils/hase/hdgwas/hdregression.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import numpy as np
2-
from utils.hase.hdgwas.tools import Timer
2+
3+
from GenNet_utils.hase.hdgwas.tools import Timer
34

45

56
# @timing

‎utils/hase/hdgwas/pard.py ‎GenNet_utils/hase/hdgwas/pard.py

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
1-
from utils.hase.hdgwas.hdregression import HASE, A_covariates, A_tests, B_covariates, C_matrix, A_inverse, B4
2-
from utils.hase.hdgwas.tools import study_indexes, Timer
3-
import numpy as np
41
import os
52
import time
63

4+
import numpy as np
5+
6+
from GenNet_utils.hase.hdgwas.hdregression import A_covariates, A_tests, B_covariates, C_matrix, B4
7+
from GenNet_utils.hase.hdgwas.tools import study_indexes, Timer
8+
79

810
def merge_PD(path, max_node, study_name):
911
print('Merging PD...')

‎utils/hase/hdgwas/protocol.py ‎GenNet_utils/hase/hdgwas/protocol.py

-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
import os
2-
import sys
3-
import string
42

53

64
class Protocol: # TODO (mid) write class

‎utils/hase/hdgwas/regression.py ‎GenNet_utils/hase/hdgwas/regression.py

+6-11
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,11 @@
1-
import h5py
21
import numpy as np
3-
import pandas as pd
4-
import os
5-
import sys
62
import gc
7-
from utils.hase.hdgwas.tools import study_indexes, Mapper, HaseAnalyser, Timer, merge_genotype
8-
from utils.hase.hdgwas.data import MetaParData
9-
from utils.hase.hdgwas.hdregression import HASE, A_covariates, A_tests, B_covariates, C_matrix, A_inverse, B4
10-
from scipy import stats
11-
import bitarray as ba
12-
from utils.hase.hdgwas.pard import partial_derivatives
13-
import tables
3+
import os
4+
5+
import numpy as np
6+
7+
from GenNet_utils.hase.hdgwas.hdregression import HASE, A_covariates, A_tests, B_covariates, C_matrix, A_inverse, B4
8+
from GenNet_utils.hase.hdgwas.tools import study_indexes, Timer, merge_genotype
149

1510

1611
def haseregression(phen, gen, cov, mapper, Analyser, maf, intercept=True, interaction=None):

0 commit comments

Comments
 (0)