Skip to content

Commit dd23064

Browse files
fzakirovfzakirov
committed
Added scripts
1 parent c24da23 commit dd23064

File tree

1 file changed

+249
-0
lines changed

1 file changed

+249
-0
lines changed

‎code/file_wrangling/concatenate_tf.ipynb‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 15,
6+
"id": "c77c88b8-2a44-4cbb-92db-7e4af858bc75",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import mne\n",
11+
"import pandas as pd\n",
12+
"import matplotlib.pyplot as plt\n",
13+
"import numpy as np\n",
14+
"from glob import glob\n",
15+
"import scipy.io\n",
16+
"import h5py\n",
17+
"import os\n",
18+
"from tqdm import tqdm"
19+
]
20+
},
21+
{
22+
"cell_type": "markdown",
23+
"id": "7429d1aa-cd96-4d79-af58-27dc4e85eaf0",
24+
"metadata": {},
25+
"source": [
26+
"## Concatenate TF"
27+
]
28+
},
29+
{
30+
"cell_type": "code",
31+
"execution_count": 147,
32+
"id": "70aab6cb-cf53-456b-acd0-4a67c2fb71fa",
33+
"metadata": {},
34+
"outputs": [
35+
{
36+
"data": {
37+
"text/plain": [
38+
"132"
39+
]
40+
},
41+
"execution_count": 147,
42+
"metadata": {},
43+
"output_type": "execute_result"
44+
}
45+
],
46+
"source": [
47+
"csv_path = f\"derivatives/behavior/\"\n",
48+
"output_path = f\"derivatives/preprocessed/TF_arrays/\"\n",
49+
"\n",
50+
"# take IDs from fully processed behavioral data (checked for accuracy, validRT, missed responses) separately for each condition\n",
51+
"sub_nonsoc = list(pd.read_csv(glob(f\"{csv_path}thrive_data_nonsoc.csv\")[0])[\"sub\"])\n",
52+
"sub_soc = list(pd.read_csv(glob(f\"{csv_path}thrive_data_soc.csv\")[0])[\"sub\"])\n",
53+
"\n",
54+
"# tf_files = sorted(glob(f\"{data_path}/sub-*{condition}*.mat\"))\n",
55+
"for measure in [\n",
56+
" \"TF\",\n",
57+
" \"ITPS\",\n",
58+
" \"ICPS\",\n",
59+
" \"wPLI\"\n",
60+
"]:\n",
61+
" if measure == \"ITPS\" or measure == \"ICPS\":\n",
62+
" key_idx = 1\n",
63+
" else:\n",
64+
" key_idx = -1\n",
65+
" data_path = f\"derivatives/preprocessed/TF_outputs/main/resp/{measure}/\"\n",
66+
" for condition in tqdm([\"resp_ns_c_1\", \"resp_ns_i_0\", \"resp_ns_i_1\",\n",
67+
" \"resp_s_i_1\", \"resp_s_c_1\", \"resp_s_i_0\"]):\n",
68+
" if condition.split(\"_\")[1] == \"s\":\n",
69+
" valid_sub_list = sub_soc.copy()\n",
70+
" elif condition.split(\"_\")[1] == \"ns\":\n",
71+
" valid_sub_list = sub_nonsoc.copy()\n",
72+
" \n",
73+
" arr_list = []\n",
74+
" subjects_with_data = [] \n",
75+
" for sub_id in valid_sub_list:\n",
76+
" try:\n",
77+
" tf_files = sorted(glob(f\"{data_path}/sub-{sub_id}*{measure}*{condition}*.mat\"))\n",
78+
" # if len(tf_files) == 0:\n",
79+
" # print(f\"{sub_id} not in TF\")\n",
80+
" data_file = h5py.File(tf_files[0])\n",
81+
" key_list = list(data_file.keys())\n",
82+
" data = data_file[key_list[key_idx]]\n",
83+
" assert data.shape == (64, 375, 59), \"Check your data!\"\n",
84+
" arr_list.append(data)\n",
85+
" subjects_with_data.append(sub_id)\n",
86+
" except: continue\n",
87+
" \n",
88+
" full_data = np.stack(arr_list, axis=0)\n",
89+
" assert full_data.shape[0] == len(subjects_with_data), \"Check your data!\"\n",
90+
" len(arr_list)\n",
91+
" scipy.io.savemat(f\"{output_path}/{measure}_{condition}.mat\",\n",
92+
" {\n",
93+
" f\"{measure}_{condition}\": full_data,\n",
94+
" f\"subjects\": subjects_with_data,\n",
95+
" })"
96+
]
97+
},
98+
{
99+
"cell_type": "markdown",
100+
"id": "ddfa7622-4174-4289-9185-e1500d1b5e4b",
101+
"metadata": {},
102+
"source": [
103+
"## Inspect number of events"
104+
]
105+
},
106+
{
107+
"cell_type": "code",
108+
"execution_count": null,
109+
"id": "546b567f-71fd-4d39-b9ed-b5fe5d794584",
110+
"metadata": {},
111+
"outputs": [],
112+
"source": [
113+
"import time\n",
114+
"sub_to_inspect = \"192\"\n",
115+
"trial_data = dict({\n",
116+
" \"sub\": [],\n",
117+
" \"s_resp_incon_error\": [],\n",
118+
" \"s_resp_incon_corr\": [],\n",
119+
" \"ns_resp_incon_error\": [],\n",
120+
" \"ns_resp_incon_corr\": [],\n",
121+
" \"s_stim_incon_corr\": [],\n",
122+
" \"s_stim_con_corr\": [],\n",
123+
" \"ns_stim_incon_corr\": [],\n",
124+
" \"ns_stim_con_corr\": [],\n",
125+
"})\n",
126+
"\n",
127+
"dataset_path = \"/home/data/NDClab/datasets/thrive-dataset/\"\n",
128+
"\n",
129+
"sub_ids = sorted([i.split(\"/\")[-1] for i in glob(\n",
130+
" f\"{dataset_path}derivatives/preprocessed/sub-*{sub_to_inspect}*\")])\n",
131+
"\n",
132+
"list_of_eeg_file = sorted(\n",
133+
" glob(\n",
134+
" f\"{dataset_path}derivatives/preprocessed/*{sub_to_inspect}*/s1_r1/eeg/*all_eeg_processed_data*.set\")\n",
135+
")\n",
136+
"\n",
137+
"start = time.time()\n",
138+
"\n",
139+
"for file_idx, filename in enumerate(list_of_eeg_file):\n",
140+
" sub_id = sub_ids[file_idx].split(\"-\")[-1]\n",
141+
" trial_data[\"sub\"].append(sub_id)\n",
142+
" EEG = scipy.io.loadmat(filename, squeeze_me=True, struct_as_record=False)[\"EEG\"]\n",
143+
" EEG_mne = mne.io.read_epochs_eeglab(filename, verbose = 'ERROR',)\n",
144+
" \n",
145+
" events = EEG.event\n",
146+
" n_times = EEG.pnts\n",
147+
" sr = EEG.srate\n",
148+
" num_ch = EEG.nbchan\n",
149+
"\n",
150+
" drop_idx = []\n",
151+
" for i in range(len(events)):\n",
152+
" latency = eeg_point2lat(\n",
153+
" [events[i].latency],\n",
154+
" [events[i].epoch],\n",
155+
" sr,\n",
156+
" timewin = [EEG.xmin*1000, EEG.xmax*1000],\n",
157+
" timeunit = 1e-3,\n",
158+
" )\n",
159+
" if latency >= -.1 and latency <= .1:\n",
160+
" drop_idx.append(i)\n",
161+
" \n",
162+
" events = [ev for ev in events if list(events).index(ev) in drop_idx]\n",
163+
" print(f\"sub-{sub_id}: {len(events)} good events were found!\")\n",
164+
" \n",
165+
" trial_data[\"s_resp_incon_error\"].append(len(\n",
166+
" [ev for ev in events if\\\n",
167+
" (ev.observation == \"s\") & (ev.eventType == \"resp\") & (ev.congruency == \"i\")\\\n",
168+
" & (ev.accuracy == 0) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
169+
" ]\n",
170+
" ))\n",
171+
" \n",
172+
" trial_data[\"s_resp_incon_corr\"].append(len(\n",
173+
" [ev for ev in events if\\\n",
174+
" (ev.observation == \"s\") & (ev.eventType == \"resp\") & (ev.congruency == \"i\")\\\n",
175+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
176+
" ]\n",
177+
" ))\n",
178+
" \n",
179+
" trial_data[\"ns_resp_incon_error\"].append(len(\n",
180+
" [ev for ev in events if\\\n",
181+
" (ev.observation == \"ns\") & (ev.eventType == \"resp\") & (ev.congruency == \"i\")\\\n",
182+
" & (ev.accuracy == 0) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
183+
" ]\n",
184+
" ))\n",
185+
" \n",
186+
" trial_data[\"ns_resp_incon_corr\"].append(len(\n",
187+
" [ev for ev in events if\\\n",
188+
" (ev.observation == \"ns\") & (ev.eventType == \"resp\") & (ev.congruency == \"i\")\\\n",
189+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
190+
" ]\n",
191+
" ))\n",
192+
" \n",
193+
" trial_data[\"s_stim_incon_corr\"].append(len(\n",
194+
" [ev for ev in events if\\\n",
195+
" (ev.observation == \"s\") & (ev.eventType == \"stim\") & (ev.congruency == \"i\")\\\n",
196+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
197+
" ]\n",
198+
" ))\n",
199+
" \n",
200+
" trial_data[\"s_stim_con_corr\"].append(len(\n",
201+
" [ev for ev in events if\\\n",
202+
" (ev.observation == \"s\") & (ev.eventType == \"stim\") & (ev.congruency == \"c\")\\\n",
203+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
204+
" ]\n",
205+
" ))\n",
206+
" \n",
207+
" trial_data[\"ns_stim_incon_corr\"].append(len(\n",
208+
" [ev for ev in events if\\\n",
209+
" (ev.observation == \"ns\") & (ev.eventType == \"stim\") & (ev.congruency == \"i\")\\\n",
210+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
211+
" ]\n",
212+
" ))\n",
213+
" \n",
214+
" trial_data[\"ns_stim_con_corr\"].append(len(\n",
215+
" [ev for ev in events if\\\n",
216+
" (ev.observation == \"ns\") & (ev.eventType == \"stim\") & (ev.congruency == \"c\")\\\n",
217+
" & (ev.accuracy == 1) & (ev.responded == 1) & (ev.validRt == 1) & (ev.extraResponse == 0)\n",
218+
" ]\n",
219+
" ))\n",
220+
"\n",
221+
"end = time.time()\n",
222+
"print(f\"Executed time {np.round(end - start, 2)} s\")\n",
223+
"\n",
224+
"pd.DataFrame(trial_data)"
225+
]
226+
}
227+
],
228+
"metadata": {
229+
"kernelspec": {
230+
"display_name": "Python 3 (ipykernel)",
231+
"language": "python",
232+
"name": "python3"
233+
},
234+
"language_info": {
235+
"codemirror_mode": {
236+
"name": "ipython",
237+
"version": 3
238+
},
239+
"file_extension": ".py",
240+
"mimetype": "text/x-python",
241+
"name": "python",
242+
"nbconvert_exporter": "python",
243+
"pygments_lexer": "ipython3",
244+
"version": "3.12.5"
245+
}
246+
},
247+
"nbformat": 4,
248+
"nbformat_minor": 5
249+
}

0 commit comments

Comments
 (0)