klustakwik2/dev/testing_distributed.py at e3dc6e602874ad71a3fd8185dddf6dd48316f677 · kwikteam/klustakwik2 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from pylab import *
from klustakwik2 import *
import cPickle as pickle
import os
from ipyparallel import Client

fname, shank = '../temp/testsmallish', 4
params = dict(
    max_iterations=1000,
    use_mua_cluster=False,
#             split_every=1, split_first=1, # for debugging splits
#             split_every=1000000, split_first=1000000, # disable splitting
#             points_for_cluster_mask=1e-100, # don't use reduced cluster masks
#             full_step_every=1,
#             always_split_bimodal=True,
#             dist_thresh=15,
#             subset_break_fraction=0.01,
#             break_fraction=0.01,
#             fast_split=True,
#             max_split_iterations=10,
    consider_cluster_deletion=True,
    )
num_starting_masks = 100

################################################
iterations = []
scores = []
num_clusters = []

log_to_file(fname+'.klg.'+str(shank), 'debug')
log_suppress_hierarchy('klustakwik', inclusive=False)

if os.path.exists(fname+'.pickle'):
    data = pickle.load(open(fname+'.pickle', 'rb'))
else:
    raw_data = load_fet_fmask_to_raw(fname, shank, drop_last_n_features=1)
    data = raw_data.to_sparse_data()
    pickle.dump(data, open(fname+'.pickle', 'wb'), -1)

client = Client()
distributer = IPythonDistributer(client)
#distributer = None

kk = KK(data, distributer=distributer, **params)
kk.register_callback(SaveCluEvery(fname, shank, every=5))

if False:
    clusters = loadtxt(fname+'.clu.'+str(shank), skiprows=1, dtype=int)
    kk.cluster_from(clusters)
else:
    #kk.cluster_with_subset_schedule(100, [0.99, 1.0])
    kk.cluster_mask_starts()

save_clu(kk, fname, shank)