-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathProcessing.R
1554 lines (1342 loc) · 83.3 KB
/
Processing.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
## ---------------------------
##
## Script name: PreProcessing
##
## Purpose of script: Metabolomics (raw ion counts) pre-processing, normalization, outlier detection and QC plots
##
## Author: Dimitrios Prymidis and Christina Schmidt
##
## Date Created: 2022-10-28
##
## Copyright (c) Dimitrios Prymidis and Christina Schmidt
## Email:
##
## ---------------------------
##
## Notes:
##
##
## ---------------------------
###################################################
### ### ### Metabolomics pre-processing ### ### ###
###################################################
#' Modularised Normalization: 80%-filtering rule, total-ion count normalization, missing value imputation and Outlier Detection: HotellingT2.
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected.
#' @param SettingsFile_Sample DF which contains information about the samples, which will be combined with the input data based on the unique sample identifiers used as rownames.
#' @param SettingsInfo Named vector containing the information about the names of the experimental parameters. c(Conditions="ColumnName_Plot_SettingsFile", Biological_Replicates="ColumnName_Plot_SettingsFile"). Column "Conditions" with information about the sample conditions (e.g. "N" and "T" or "Normal" and "Tumor"), can be used for feature filtering and colour coding in the PCA. Column "BiologicalReplicates" including numerical values. For CoRe = TRUE a CoRe_norm_factor = "Columnname_Input_SettingsFile" and CoRe_media = "Columnname_Input_SettingsFile", have to also be added. Column CoRe_norm_factor is used for normalization and CoRe_media is used to specify the name of the media controls in the Conditions.
#' @param FeatureFilt \emph{Optional: }If NULL, no feature filtering is performed. If set to "Standard" then it applies the 80%-filtering rule (Bijlsma S. et al., 2006) on the metabolite features on the whole dataset. If is set to "Modified",filtering is done based on the different conditions, thus a column named "Conditions" must be provided in the Input_SettingsFile input file including the individual conditions you want to apply the filtering to (Yang, J et al., 2015). \strong{Default = "Standard"}
#' @param FeatureFilt_Value \emph{Optional: } Percentage of feature filtering. \strong{Default = 0.8}
#' @param TIC \emph{Optional: } If TRUE, Total Ion Count normalization is performed. \strong{Default = TRUE}
#' @param MVI \emph{Optional: } If TRUE, Missing Value Imputation (MVI) based on half minimum is performed \strong{Default = TRUE}
#' @param MVI_Percentage \emph{Optional: } Percentage 0-100 of imputed value based on the minimum value. \strong{Default = 50}
#' @param HotellinsConfidence \emph{Optional: } Defines the Confidence of Outlier identification in HotellingT2 test. Must be numeric.\strong{Default = 0.99}
#' @param CoRe \emph{Optional: } If TRUE, a consumption-release experiment has been performed and the CoRe value will be calculated. Please consider providing a Normalisation factor column called "CoRe_norm_factor" in your "Input_SettingsFile" DF, where the column "Conditions" matches. The normalisation factor must be a numerical value obtained from growth rate that has been obtained from a growth curve or growth factor that was obtained by the ratio of cell count/protein quantification at the start point to cell count/protein quantification at the end point.. Additionally control media samples have to be available in the "Input" DF and defined as "CoRe_media" samples in the "Conditions" column in the "Input_SettingsFile" DF. \strong{Default = FALSE}
#' @param SaveAs_Plot \emph{Optional: } Select the file type of output plots. Options are svg, png, pdf. If set to NULL, plots are not saved. \strong{Default = svg}
#' @param SaveAs_Table \emph{Optional: } Select the file type of output table. Options are "csv", "xlsx", "txt". If set to NULL, plots are not saved. \strong{Default = "csv"}
#' @param PrintPlot \emph{Optional: } If TRUE prints an overview of resulting plots. \strong{Default = TRUE}
#' @param FolderPath \emph{Optional:} Path to the folder the results should be saved at. \strong{default: NULL}
#'
#' @return List with two elements: DF (including all output tables generated) and Plot (including all plots generated)
#'
#' @examples
#' Intra <- MetaProViz::ToyData("IntraCells_Raw")
#' ResI <- MetaProViz::PreProcessing(InputData=Intra[-c(49:58) ,-c(1:3)],
#' SettingsFile_Sample=Intra[-c(49:58) , c(1:3)],
#' SettingsInfo = c(Conditions = "Conditions", Biological_Replicates = "Biological_Replicates"))
#'
#' Media <- MetaProViz::ToyData("CultureMedia_Raw")
#' ResM <- MetaProViz::PreProcessing(InputData = Media[-c(40:45) ,-c(1:3)],
#' SettingsFile_Sample = Media[-c(40:45) ,c(1:3)] ,
#' SettingsInfo = c(Conditions = "Conditions", Biological_Replicates = "Biological_Replicates", CoRe_norm_factor = "GrowthFactor", CoRe_media = "blank"),
#' CoRe=TRUE)
#'
#' @keywords 80 percent filtering rule, Missing Value Imputation, Total Ion Count normalization, PCA, HotellingT2, multivariate quality control charts
#'
#' @importFrom dplyr mutate_all
#' @importFrom magrittr %>% %<>%
#' @importFrom tibble rownames_to_column column_to_rownames
#'
#' @export
#'
PreProcessing <- function(InputData,
SettingsFile_Sample,
SettingsInfo,
FeatureFilt = "Modified",
FeatureFilt_Value = 0.8,
TIC = TRUE,
MVI= TRUE,
MVI_Percentage=50,
HotellinsConfidence = 0.99,
CoRe = FALSE,
SaveAs_Plot = "svg",
SaveAs_Table = "csv",
PrintPlot = TRUE,
FolderPath = NULL
){
## ------------ Create log file ----------- ##
MetaProViz_Init()
## ------------------ Check Input ------------------- ##
# HelperFunction `CheckInput`
CheckInput(InputData=InputData,
SettingsFile_Sample=SettingsFile_Sample,
SettingsFile_Metab=NULL,
SettingsInfo= SettingsInfo,
SaveAs_Plot=SaveAs_Plot,
SaveAs_Table=SaveAs_Table,
CoRe=CoRe,
PrintPlot= PrintPlot)
# HelperFunction `CheckInput` Specific
CheckInput_PreProcessing(SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
CoRe=CoRe,
FeatureFilt=FeatureFilt,
FeatureFilt_Value=FeatureFilt_Value,
TIC=TIC,
MVI=MVI,
MVI_Percentage=MVI_Percentage,
HotellinsConfidence=HotellinsConfidence)
## ------------------ Create output folders and path ------------------- ##
if(is.null(SaveAs_Plot)==FALSE |is.null(SaveAs_Table)==FALSE ){
Folder <- SavePath(FolderName= "Processing",
FolderPath=FolderPath)
SubFolder_P <- file.path(Folder, "PreProcessing")
if (!dir.exists(SubFolder_P)) {dir.create(SubFolder_P)}
}
## ------------------ Prepare the data ------------------- ##
#InputData files:
InputData <-as.data.frame(InputData)%>%
dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))#Make sure all 0 are changed to NAs
InputData <- as.data.frame(dplyr::mutate_all(as.data.frame(InputData), function(x) as.numeric(as.character(x))))
###################################################################################################################################
## ------------------ 1. Feature filtering ------------------- ##
if(is.null(FeatureFilt)==FALSE){
InputData_Filtered <- FeatureFiltering(InputData=InputData,
FeatureFilt=FeatureFilt,
FeatureFilt_Value=FeatureFilt_Value,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
CoRe=CoRe)
InputData_Filt <- InputData_Filtered[["DF"]]
}else{
InputData_Filt <- InputData
}
## ------------------ 2. Missing value Imputation ------------------- ##
if(MVI==TRUE){
MVIRes<- MVImputation(InputData=InputData_Filt,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
CoRe=CoRe,
MVI_Percentage=MVI_Percentage)
}else{
MVIRes<- InputData_Filt
}
## ------------------ 3. Total Ion Current Normalization ------------------- ##
if(TIC==TRUE){
#Perform TIC
TICRes_List <- TICNorm(InputData=MVIRes,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
TIC=TIC)
TICRes <- TICRes_List[["DF"]][["Data_TIC"]]
#Add plots to PlotList
PlotList <- list()
PlotList[["RLAPlot"]] <- TICRes_List[["Plot"]][["RLA_BeforeTICNorm"]]
PlotList[["RLAPlot_TICnorm"]] <- TICRes_List[["Plot"]][["RLA_AfterTICNorm"]]
PlotList[["RLAPlot_BeforeAfter_TICnorm"]] <- TICRes_List[["Plot"]][["norm_plots"]]
}else{
TICRes <- MVIRes
#Add plots to PlotList
RLAPlot_List <- TICNorm(InputData=MVIRes,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
TIC=TIC)
PlotList <- list()
PlotList[["RLAPlot"]] <- RLAPlot_List[["Plot"]][["RLA_BeforeTICNorm"]]
}
## ------------------ 4. CoRe media QC (blank) and normalization ------------------- ##
if(CoRe ==TRUE){
data_CoReNorm <- CoReNorm(InputData= TICRes,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo)
TICRes <- data_CoReNorm[["DF"]][["Core_Norm"]]
}
# ------------------ Final Output:
data_norm <- TICRes %>% as.data.frame()
###################################################################################################################################
## ------------------ Sample outlier identification ------------------- ##
OutlierRes <- OutlierDetection(InputData= data_norm,
SettingsFile_Sample=SettingsFile_Sample,
SettingsInfo=SettingsInfo,
CoRe=CoRe,
HotellinsConfidence=HotellinsConfidence)
###################################################################################################################################
## ------------------ Return ------------------- ##
## ---- DFs
if(is.null(FeatureFilt)==FALSE){#Add metabolites that where removed as part of the feature filtering
if(length(InputData_Filtered[["RemovedMetabolites"]])==0){
DFList <- list("InputData_RawData"= merge(as.data.frame(SettingsFile_Sample), as.data.frame(InputData), by="row.names")%>% tibble::column_to_rownames("Row.names"),
"Filtered_metabolites"= as.data.frame(list(FeatureFiltering = c(FeatureFilt),
FeatureFilt_Value = c(FeatureFilt_Value),
RemovedMetabolites = c("None"))),
"Preprocessing_output"=OutlierRes[["DF"]][["data_outliers"]])
}else{
DFList <- list("InputData_RawData"= merge(as.data.frame(SettingsFile_Sample), as.data.frame(InputData), by="row.names")%>% tibble::column_to_rownames("Row.names"),
"Filtered_metabolites"= as.data.frame(list(FeatureFiltering = rep(FeatureFilt, length(InputData_Filtered[["RemovedMetabolites"]])),
FeatureFilt_Value = rep(FeatureFilt_Value, length(InputData_Filtered[["RemovedMetabolites"]])),
RemovedMetabolites = InputData_Filtered[["RemovedMetabolites"]])),
"Preprocessing_output"=OutlierRes[["DF"]][["data_outliers"]])
}
}else{
DFList <- list("InputData_RawData"= merge(as.data.frame(SettingsFile_Sample), as.data.frame(InputData), by="row.names")%>% tibble::column_to_rownames("Row.names"), "Preprocessing_output"=OutlierRes[["DF"]][["data_outliers"]])
}
if(CoRe ==TRUE){
if(is.null(data_CoReNorm[["DF"]][["Contigency_table_CoRe_blank"]])){
DFList_CoRe <- list( "CV_CoRe_blank"= data_CoReNorm[["DF"]][["CV_CoRe_blank"]])
}else{
DFList_CoRe <- list( "CV_CoRe_blank"= data_CoReNorm[["DF"]][["CV_CoRe_blank"]],"Variation_ContigencyTable_CoRe_blank"=data_CoReNorm[["DF"]][["Contigency_table_CoRe_blank"]])
}
DFList <- c(DFList, DFList_CoRe)
}
## ---- Plots
if(TIC==TRUE){
PlotList <- c(TICRes_List[["Plot"]], OutlierRes[["Plot"]])
}else{
PlotList <- c(RLAPlot_List[["Plot"]], OutlierRes[["Plot"]])
}
if(CoRe ==TRUE){
PlotList <- c(PlotList , data_CoReNorm[["Plot"]])
}
Res_List <- list("DF"= DFList ,"Plot" =PlotList)
# Save Plots and DFs
#As row names are not saved we need to make row.names to column for the DFs that needs this:
DFList[["InputData_RawData"]] <- DFList[["InputData_RawData"]]%>%tibble::rownames_to_column("Code")
DFList[["Preprocessing_output"]] <- DFList[["Preprocessing_output"]]%>%tibble::rownames_to_column("Code")
suppressMessages(suppressWarnings(
SaveRes(InputList_DF=DFList,
InputList_Plot= PlotList,
SaveAs_Table=SaveAs_Table,
SaveAs_Plot=SaveAs_Plot,
FolderPath= SubFolder_P,
FileName= "PreProcessing",
CoRe=CoRe,
PrintPlot=PrintPlot)))
#Return
invisible(return(Res_List))
}
############################################################
### ### ### Merge analytical replicates function ### ### ###
############################################################
#' Merges the analytical replicates of an experiment
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected.
#' @param SettingsFile_Sample DF which contains information about the samples Column "Conditions", "Biological_replicates" and "Analytical_Replicates has to exist.
#' @param SettingsInfo \emph{Optional: } Named vector including the Conditions and Replicates information: c(Conditions="ColumnNameConditions", Biological_Replicates="ColumnName_SettingsFile_Sample", Analytical_Replicates="ColumnName_SettingsFile_Sample").\strong{Default = NULL}
#' @param SaveAs_Table \emph{Optional: } File types for the analysis results are: "csv", "xlsx", "txt", ot NULL \strong{default: "csv"}
#' @param FolderPath \emph{Optional:} Path to the folder the results should be saved at. \strong{default: NULL}
#'
#' @return DF with the merged analytical replicates
#'
#' @examples
#' Intra <- ToyData("IntraCells_Raw")
#' Res <- ReplicateSum(InputData=Intra[-c(49:58) ,-c(1:3)],
#' SettingsFile_Sample=Intra[-c(49:58) , c(1:3)],
#' SettingsInfo = c(Conditions="Conditions", Biological_Replicates="Biological_Replicates", Analytical_Replicates="Analytical_Replicates"))
#'
#' @keywords Analytical Replicate Merge
#'
#' @importFrom dplyr mutate_all summarise_all select rename ungroup group_by
#' @importFrom magrittr %>% %<>%
#' @importFrom tibble rownames_to_column column_to_rownames
#' @importFrom rlang !! :=
#' @importFrom tidyr unite
#'
#' @export
#'
ReplicateSum <- function(InputData,
SettingsFile_Sample,
SettingsInfo = c(Conditions="Conditions", Biological_Replicates="Biological_Replicates", Analytical_Replicates="Analytical_Replicates"),
SaveAs_Table = "csv",
FolderPath = NULL){
## ------------ Create log file ----------- ##
MetaProViz_Init()
## ------------------ Check Input ------------------- ##
# HelperFunction `CheckInput`
CheckInput(InputData=InputData,
SettingsFile_Sample=SettingsFile_Sample,
SettingsFile_Metab=NULL,
SettingsInfo = SettingsInfo,
SaveAs_Plot=NULL,
SaveAs_Table=SaveAs_Table,
CoRe=FALSE,
PrintPlot=FALSE)
# `CheckInput` Specific
if(SettingsInfo[["Conditions"]] %in% colnames(SettingsFile_Sample)){
# Conditions <- InputData[[SettingsInfo[["Conditions"]] ]]
}else{
stop("Column `Conditions` is required.")
}
if(SettingsInfo[["Biological_Replicates"]] %in% colnames(SettingsFile_Sample)){
#Biological_Replicates <- InputData[[SettingsInfo[["Biological_Replicates"]]]]
}else{
stop("Column `Biological_Replicates` is required.")
}
if(SettingsInfo[["Analytical_Replicates"]] %in% colnames(SettingsFile_Sample)){
#Analytical_Replicates <- InputData[[SettingsInfo[["Analytical_Replicates"]]]]
}else{
stop("Column `Analytical_Replicates` is required.")
}
## ------------ Create Results output folder ----------- ##
if(is.null(SaveAs_Table)==FALSE ){
Folder <- SavePath(FolderName= "Processing",
FolderPath=FolderPath)
SubFolder <- file.path(Folder, "ReplicateSum")
if (!dir.exists(SubFolder)) {dir.create(SubFolder)}
}
## ------------ Load data and process ----------- ##
Input <- merge(x= SettingsFile_Sample%>% dplyr::select(!!SettingsInfo[["Conditions"]], !!SettingsInfo[["Biological_Replicates"]], !!SettingsInfo[["Analytical_Replicates"]]),
y= InputData,
by="row.names")%>%
tibble::column_to_rownames("Row.names")%>%
dplyr::rename("Conditions"=SettingsInfo[["Conditions"]],
"Biological_Replicates"=SettingsInfo[["Biological_Replicates"]],
"Analytical_Replicates"=SettingsInfo[["Analytical_Replicates"]])
# Make the replicate Sums
Input_data_numeric_summed <- as.data.frame(Input %>%
dplyr::group_by(Biological_Replicates, Conditions) %>%
dplyr::summarise_all("mean") %>% dplyr::select(-Analytical_Replicates))
# Make a number of merged replicates column
nReplicates <- Input %>%
dplyr::group_by(Biological_Replicates, Conditions) %>%
dplyr::summarise_all("max") %>%
dplyr::ungroup() %>%
dplyr::select(Analytical_Replicates, Biological_Replicates, Conditions) %>%
dplyr::rename("n_AnalyticalReplicates_Summed "= "Analytical_Replicates")
Input_data_numeric_summed <- merge(nReplicates,Input_data_numeric_summed, by = c("Conditions","Biological_Replicates"))%>%
tidyr::unite(UniqueID, c("Conditions","Biological_Replicates"), sep="_", remove=FALSE)%>% # Create a uniqueID
tibble::column_to_rownames("UniqueID")# set UniqueID to rownames
#--------------- return ------------------##
SaveRes(InputList_DF=list("Sum_AnalyticalReplicates"=Input_data_numeric_summed%>%tibble::rownames_to_column("Code")),
InputList_Plot = NULL,
SaveAs_Table=SaveAs_Table,
SaveAs_Plot=NULL,
FolderPath= SubFolder,
FileName= "Sum_AnalyticalReplicates",
CoRe=FALSE,
PrintPlot=FALSE)
#Return
invisible(return(Input_data_numeric_summed))
}
##########################################################################
### ### ### Metabolite detection estimation using pool samples ### ### ###
##########################################################################
#' Find metabolites with high variability across total pool samples
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected. Can be either a full dataset or a dataset with only the pool samples.
#' @param SettingsFile_Sample \emph{Optional: } DF which contains information about the samples when a full dataset is inserted as Input_data. Column "Conditions" with information about the sample conditions (e.g. "N" and "T" or "Normal" and "Tumor"), has to exist.\strong{Default = NULL}
#' @param SettingsInfo \emph{Optional: } NULL or Named vector including the Conditions and PoolSample information (Name of the Conditions column and Name of the pooled samples in the Conditions in the Input_SettingsFile) : c(Conditions="ColumnNameConditions, PoolSamples=NamePoolCondition. If no Conditions is added in the Input_SettingsInfo, it is assumed that the conditions column is named 'Conditions' in the Input_SettingsFile. ). \strong{Default = NULL}
#' @param CutoffCV \emph{Optional: } Filtering cutoff for high variance metabolites using the Coefficient of Variation. \strong{Default = 30}
#' @param SaveAs_Plot \emph{Optional: } Select the file type of output plots. Options are svg, png, pdf or NULL. \strong{Default = svg}
#' @param SaveAs_Table \emph{Optional: } File types for the analysis results are: "csv", "xlsx", "txt", ot NULL \strong{default: "csv"}
#' @param PrintPlot \emph{Optional: } If TRUE prints an overview of resulting plots. \strong{Default = TRUE}
#' @param FolderPath \emph{Optional:} Path to the folder the results should be saved at. \strong{default: NULL}
#'
#' @return List with two elements: DF (including input and output table) and Plot (including all plots generated)
#'
#' @examples
#' Intra <- ToyData("IntraCells_Raw")
#' Res <- PoolEstimation(InputData=Intra[ ,-c(1:3)],
#' SettingsFile_Sample=Intra[ , c(1:3)],
#' SettingsInfo = c(PoolSamples = "Pool", Conditions="Conditions"))
#'
#' @keywords Coefficient of Variation, high variance metabolites
#'
#' @importFrom dplyr case_when select rowwise mutate ungroup
#' @importFrom magrittr %>% %<>%
#' @importFrom tibble rownames_to_column column_to_rownames
#' @importFrom logger log_info log_trace
#' @importFrom ggplot2 after_stat
#'
#' @export
#'
PoolEstimation <- function(InputData,
SettingsFile_Sample = NULL,
SettingsInfo = NULL,
CutoffCV = 30,
SaveAs_Plot = "svg",
SaveAs_Table = "csv",
PrintPlot=TRUE,
FolderPath = NULL){
## ------------ Create log file ----------- ##
MetaProViz_Init()
logger::log_info('Starting pool estimation.')
## ------------------ Check Input ------------------- ##
# HelperFunction `CheckInput`
CheckInput(InputData=InputData,
SettingsFile_Sample=SettingsFile_Sample,
SettingsFile_Metab=NULL,
SettingsInfo=SettingsInfo,
SaveAs_Plot=SaveAs_Plot,
SaveAs_Table=SaveAs_Table,
CoRe=FALSE,
PrintPlot = PrintPlot)
# `CheckInput` Specific
if(is.null(SettingsFile_Sample)==FALSE){
if("Conditions" %in% names(SettingsInfo)==TRUE){
if(SettingsInfo[["Conditions"]] %in% colnames(SettingsFile_Sample)== FALSE ){
stop("You have chosen Conditions = ",paste(SettingsInfo[["Conditions"]]), ", ", paste(SettingsInfo[["Conditions"]])," was not found in SettingsFile_Sample as column. Please insert the name of the experimental conditions as stated in the SettingsFile_Sample." )
}
}
if("PoolSamples" %in% names(SettingsInfo)==TRUE){
if(SettingsInfo[["PoolSamples"]] %in% SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] == FALSE ){
stop("You have chosen PoolSamples = ",paste(SettingsInfo[["PoolSamples"]] ), ", ", paste(SettingsInfo[["PoolSamples"]] )," was not found in SettingsFile_Sample as sample condition. Please insert the name of the pool samples as stated in the Conditions column of the SettingsFile_Sample." )
}
}
}
if(is.numeric(CutoffCV)== FALSE | CutoffCV < 0){
stop("Check input. The selected CutoffCV value should be a positive numeric value.")
}
## ------------------ Create output folders and path ------------------- ##
if(is.null(SaveAs_Plot)==FALSE |is.null(SaveAs_Table)==FALSE ){
Folder <- SavePath(FolderName= "Processing",
FolderPath=FolderPath)
SubFolder <- file.path(Folder, "PoolEstimation")
logger::log_info('Selected output directory: `%s`.', SubFolder)
if (!dir.exists(SubFolder)) {
logger::log_trace('Creating directory: `%s`.', SubFolder)
dir.create(SubFolder)
}
}
## ------------------ Prepare the data ------------------- ##
#InputData files:
if(is.null(SettingsFile_Sample)==TRUE){
PoolData <- InputData%>%
dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))#Make sure all 0 are changed to NAs
}else{
PoolData <- InputData[SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] == SettingsInfo[["PoolSamples"]],]%>%
dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))#Make sure all 0 are changed to NAs
}
###################################################################################################################################
## ------------------ Coefficient of Variation ------------------- ##
logger::log_trace('Calculating coefficient of variation.')
result_df <- apply(PoolData, 2, function(x) { (sd(x, na.rm =T)/ mean(x, na.rm =T))*100 } ) %>% t()%>% as.data.frame()
rownames(result_df)[1] <- "CV"
NAvector <- apply(PoolData, 2, function(x) {(sum(is.na(x))/length(x))*100 })# Calculate the NAs
# Create Output DF
result_df_final <- result_df %>%
t()%>% as.data.frame() %>% dplyr::rowwise() %>%
dplyr::mutate(HighVar = CV > CutoffCV) %>% as.data.frame()
result_df_final$MissingValuePercentage <- NAvector
rownames(result_df_final)<- colnames(InputData)
result_df_final_out <- tibble::rownames_to_column(result_df_final,"Metabolite" )
# Remove Metabolites from InputData based on CutoffCV
logger::log_trace('Applying CV cut-off.')
if(is.null(SettingsFile_Sample)==FALSE){
unstable_metabs <- rownames(result_df_final)[result_df_final[["HighVar_Metabs"]]]
if(length(unstable_metabs)>0){
filtered_Input_data <- InputData %>% dplyr::select(!unstable_metabs)
}else{
filtered_Input_data <- NULL
}
}else{
filtered_Input_data <- NULL
}
## ------------------ QC plots ------------------- ##
# Start QC plot list
logger::log_info('Plotting QC plots.')
PlotList <- list()
# 1. Pool Sample PCA
logger::log_trace('Pool sample PCA.')
dev.new()
if(is.null(SettingsFile_Sample)==TRUE){
pca_data <- PoolData
pca_QC_pool <-invisible(VizPCA(InputData=pca_data,
PlotName = "QC Pool samples",
SaveAs_Plot = NULL))
}else{
pca_data <- merge(SettingsFile_Sample %>% dplyr::select(SettingsInfo[["Conditions"]]), InputData, by=0) %>%
tibble::column_to_rownames("Row.names") %>%
dplyr::mutate(Sample_type = dplyr::case_when(.data[[SettingsInfo[["Conditions"]]]] == SettingsInfo[["PoolSamples"]] ~ "Pool",
TRUE ~ "Sample"))
pca_QC_pool <-invisible(VizPCA(InputData=pca_data %>%dplyr::select(-all_of(SettingsInfo[["Conditions"]]), -Sample_type),
SettingsInfo= c(color="Sample_type"),
SettingsFile_Sample= pca_data,
PlotName = "QC Pool samples",
SaveAs_Plot = NULL))
}
dev.off()
PlotList [["PCAPlot_PoolSamples"]] <- pca_QC_pool[["Plot_Sized"]][["Plot_Sized"]]
# 2. Histogram of CVs
logger::log_trace('CV histogram.')
HistCV <-suppressWarnings(invisible(ggplot(result_df_final_out, aes(CV)) +
geom_histogram(aes(y=ggplot2::after_stat(density)), color="black", fill="white")+
geom_vline(aes(xintercept=CutoffCV),
color="darkred", linetype="dashed", size=1)+
geom_density(alpha=.2, fill="#FF6666") +
labs(title="CV for metabolites of Pool samples",x="Coefficient of variation (CV%)", y = "Frequency")+
theme_classic()))
HistCV_Sized <- plotGrob_Processing(InputPlot = HistCV, PlotName= "CV for metabolites of Pool samples", PlotType= "Hist")
PlotList [["Histogram_CV-PoolSamples"]] <- HistCV_Sized
# 2. ViolinPlot of CVs
logger::log_trace('CV violin plot.')
#Make Violin of CVs
Plot_cv_result_df <- result_df_final_out %>%
dplyr::mutate(HighVar = ifelse((CV > CutoffCV)==TRUE, paste("> CV", CutoffCV, sep=""), paste("< CV", CutoffCV, sep="")))
ViolinCV <- invisible(ggplot( Plot_cv_result_df, aes(y=CV, x=HighVar, label=Plot_cv_result_df$Metabolite))+
geom_violin(alpha = 0.5 , fill="#FF6666")+
geom_dotplot(binaxis = "y", stackdir = "center", dotsize = 0.5) +
ggrepel::geom_text_repel(aes(label = ifelse(Plot_cv_result_df$CV > CutoffCV,
as.character(Plot_cv_result_df$Metabolite), '')),
hjust = 0, vjust = 0,
box.padding = 0.5, # space between text and point
point.padding = 0.5, # space around points
max.overlaps = Inf) + # allow for many labels
labs(title="CV for metabolites of Pool samples",x="Metabolites", y = "Coefficient of variation (CV%)")+
theme_classic())
ViolinCV_Sized <- plotGrob_Processing(InputPlot = ViolinCV, PlotName= "CV for metabolites of Pool samples", PlotType= "Violin")
PlotList [["ViolinPlot_CV-PoolSamples"]] <- ViolinCV_Sized
###################################################################################################################################
## ------------------ Return and Save ------------------- ##
#Save
logger::log_info('Preparing saved and returned data.')
if(is.null(filtered_Input_data)==FALSE){
DF_list <- list("InputData" = InputData, "Filtered_InputData" = filtered_Input_data, "CV" = result_df_final_out )
}else{
DF_list <- list("InputData" = InputData, "CV" = result_df_final_out)
}
ResList <- list("DF"= DF_list,"Plot"=PlotList)
#Save
DF_list[["InputData"]]<- DF_list[["InputData"]]%>%tibble::rownames_to_column("Code")
logger::log_info(
'Saving results: [SaveAs_Table=%s, SaveAs_Plot=%s, FolderPath=%s].',
SaveAs_Table,
SaveAs_Plot,
SubFolder
)
SaveRes(InputList_DF=DF_list,
InputList_Plot = PlotList,
SaveAs_Table=SaveAs_Table,
SaveAs_Plot=SaveAs_Plot,
FolderPath= SubFolder,
FileName= "PoolEstimation",
CoRe=FALSE,
PrintPlot=PrintPlot)
#Return
logger::log_info('Finished pool estimation.')
invisible(return(ResList))
}
################################################################################################
### ### ### PreProcessing helper function: FeatureFiltering ### ### ###
################################################################################################
#' FeatureFiltering
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected and consider converting any zeros to NA unless they are true zeros.
#' @param SettingsFile_Sample DF which contains information about the samples, which will be combined with the input data based on the unique sample identifiers used as rownames.
#' @param SettingsInfo Named vector containing the information about the names of the experimental parameters. c(Conditions="ColumnName_Plot_SettingsFile", Biological_Replicates="ColumnName_Plot_SettingsFile"). Column "Conditions" with information about the sample conditions (e.g. "N" and "T" or "Normal" and "Tumor"), can be used for feature filtering and colour coding in the PCA. Column "BiologicalReplicates" including numerical values. For CoRe = TRUE add CoRe_media = "Columnname_Input_SettingsFile", which specifies the name of the media controls in the Conditions.
#' @param CoRe \emph{Optional: } If TRUE, a consumption-release experiment has been performed.Should not be normalised to media blank. Provide information about control media sample names via SettingsInfo "CoRe_media" samples. \strong{Default = FALSE}
#' @param FeatureFilt \emph{Optional: } If NULL, no feature filtering is performed. If set to "Standard" then it applies the 80%-filtering rule (Bijlsma S. et al., 2006) on the metabolite features on the whole dataset. If is set to "Modified",filtering is done based on the different conditions, thus a column named "Conditions" must be provided in the Input_SettingsFile input file including the individual conditions you want to apply the filtering to (Yang, J et al., 2015). \strong{Default = Modified}
#' @param FeatureFilt_Value \emph{Optional: } Percentage of feature filtering. \strong{Default = 0.8}
#'
#' @return List with two elements: filtered matrix and features filtered
#'
#' @examples
#' Intra <- ToyData("IntraCells_Raw")
#' Res <- FeatureFiltering(InputData=Intra[-c(49:58), -c(1:3)]%>% dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .)),
#' SettingsFile_Sample=Intra[-c(49:58), c(1:3)],
#' SettingsInfo = c(Conditions = "Conditions", Biological_Replicates = "Biological_Replicates"))
#'
#' @keywords feature filtering or modified feature filtering
#'
#' @importFrom dplyr filter mutate_all
#' @importFrom magrittr %>% %<>%
#' @importFrom logger log_info log_trace
#'
#' @noRd
#'
FeatureFiltering <-function(InputData,
SettingsFile_Sample,
SettingsInfo,
CoRe=FALSE,
FeatureFilt="Modified",
FeatureFilt_Value=0.8){
## ------------ Create log file ----------- ##
MetaProViz_Init()
## ------------------ Prepare the data ------------------- ##
feat_filt_data <- as.data.frame(InputData)%>%
dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))#Make sure all 0 are changed to NAs
if(CoRe== TRUE){ # remove CoRe_media samples for feature filtering
feat_filt_data <- feat_filt_data %>% dplyr::filter(!SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] ==SettingsInfo[["CoRe_media"]])
Feature_Filtering <- paste0(FeatureFilt, "_CoRe")
}
## ------------------ Perform filtering ------------------ ##
if(FeatureFilt == "Modified"){
message <- paste0("FeatureFiltering: Here we apply the modified 80%-filtering rule that takes the class information (Column `Conditions`) into account, which additionally reduces the effect of missing values (REF: Yang et. al., (2015), doi: 10.3389/fmolb.2015.00004). ", "Filtering value selected: ", FeatureFilt_Value, sep="")
logger::log_info(message)
message(message)
if(CoRe== TRUE){
feat_filt_Conditions <- SettingsFile_Sample[[SettingsInfo[["Conditions"]]]][!SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] == SettingsInfo[["CoRe_media"]]]
}else{
feat_filt_Conditions <- SettingsFile_Sample[[SettingsInfo[["Conditions"]]]]
}
if(is.null(unique(feat_filt_Conditions)) == TRUE){
message("Conditions information is missing.")
logger::log_trace(message)
stop(message)
}
if(length(unique(feat_filt_Conditions)) == 1){
message("To perform the Modified feature filtering there have to be at least 2 different Conditions in the `Condition` column in the Experimental design. Consider using the Standard feature filtering option.")
logger::log_trace(message)
stop(message)
}
miss <- c()
split_Input <- split(feat_filt_data, feat_filt_Conditions) # split data frame into a list of dataframes by condition
for (m in split_Input){ # Select metabolites to be filtered for different conditions
for(i in 1:ncol(m)) {
if(length(which(is.na(m[,i]))) > (1-FeatureFilt_Value)*nrow(m))
miss <- append(miss,i)
}
}
if(length(miss) == 0){ #remove metabolites if any are found
message("There where no metabolites exluded")
filtered_matrix <- InputData
feat_file_res <- "There where no metabolites exluded"
}else{
names<-unique(colnames(InputData)[miss])
message(length(unique(miss)) ," metabolites where removed: ", paste0(names, collapse = ", "))
filtered_matrix <- InputData[,-miss]
}
}else if(FeatureFilt == "Standard"){
message <- paste0 ("FeatureFiltering: Here we apply the so-called 80%-filtering rule, which removes metabolites with missing values in more than 80% of samples (REF: Smilde et. al. (2005), Anal. Chem. 77, 6729-6736., doi:10.1021/ac051080y). ","Filtering value selected:", FeatureFilt_Value)
logger::log_info(message)
message(message)
split_Input <- feat_filt_data
miss <- c()
for(i in 1:ncol(split_Input)) { # Select metabolites to be filtered for one condition
if(length(which(is.na(split_Input[,i]))) > (1-FeatureFilt_Value)*nrow(split_Input))
miss <- append(miss,i)
}
if(length(miss) == 0){ #remove metabolites if any are found
message <- paste0("FeatureFiltering: There where no metabolites exluded")
logger::log_info(message)
message(message)
filtered_matrix <- InputData
feat_file_res <- "There where no metabolites exluded"
}else{
names<-unique(colnames(InputData)[miss])
message <- paste0(length(unique(miss)) ," metabolites where removed: ", paste0(names, collapse = ", "))
logger::log_info(message)
message(message)
filtered_matrix <- InputData[,-miss]
}
}
## ------------------ Return ------------------ ##
features_filtered <- unique(colnames(InputData)[miss]) %>% as.vector()
filtered_matrix <- as.data.frame(dplyr::mutate_all(as.data.frame(filtered_matrix), function(x) as.numeric(as.character(x))))
Filtered_results <- list("DF"= filtered_matrix , "RemovedMetabolites" = features_filtered)
invisible(return(Filtered_results))
}
################################################################################################
### ### ### PreProcessing helper function: Missing Value imputation ### ### ###
################################################################################################
#' Missing Value Imputation using half minimum value
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected and consider converting any zeros to NA unless they are true zeros.
#' @param SettingsFile_Sample DF which contains information about the samples, which will be combined with the input data based on the unique sample identifiers used as rownames.
#' @param SettingsInfo Named vector containing the information about the names of the experimental parameters. c(Conditions="ColumnName_Plot_SettingsFile", Biological_Replicates="ColumnName_Plot_SettingsFile", CoRe_media = "Columnname_Input_SettingsFile"). Column "Conditions" with information about the sample conditions, Column "BiologicalReplicates" including numerical values and Column "Columnname_Input_SettingsFile" is used to specify the name of the media controls in the Conditions.
#' @param CoRe \emph{Optional: } If TRUE, a consumption-release experiment has been performed. Should not be normalised to media blank. Provide information about control media sample names via SettingsInfo "CoRe_media" samples.\strong{Default = FALSE}
#' @param MVI_Percentage \emph{Optional: } Percentage 0-100 of imputed value based on the minimum value. \strong{Default = 50}
#'
#' @return DF with imputed values
#'
#' @examples
#' Intra <- ToyData("IntraCells_Raw")
#' Res <- MVImputation(InputData=Intra[-c(49:58), -c(1:3)]%>% dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .)),
#' SettingsFile_Sample=Intra[-c(49:58), c(1:3)],
#' SettingsInfo = c(Conditions = "Conditions", Biological_Replicates = "Biological_Replicates"))
#'
#' @keywords Half minimum missing value imputation
#'
#' @importFrom dplyr select mutate group_by filter
#' @importFrom magrittr %>% %<>%
#' @importFrom tibble column_to_rownames
#' @importFrom logger log_info log_trace
#'
#' @noRd
#'
MVImputation <-function(InputData,
SettingsFile_Sample,
SettingsInfo,
CoRe=FALSE,
MVI_Percentage=50){
## ------------------ Prepare the data ------------------- ##
filtered_matrix <- InputData%>%
dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))#Make sure all 0 are changed to NAs
## ------------------ Perform MVI ------------------ ##
# Do MVI for the samples
message <- paste0("Missing Value Imputation: Missing value imputation is performed, as a complementary approach to address the missing value problem, where the missing values are imputing using the `half minimum value`. REF: Wei et. al., (2018), Reports, 8, 663, doi:https://doi.org/10.1038/s41598-017-19120-0")
logger::log_info(message)
message(message)
if(CoRe==TRUE){#remove blank samples
NA_removed_matrix <- filtered_matrix%>% dplyr::filter(!SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] == SettingsInfo[["CoRe_media"]])
}else{
NA_removed_matrix <- filtered_matrix %>% as.data.frame()
}
for (feature in colnames(NA_removed_matrix)){
feature_data <- merge(NA_removed_matrix[feature] , SettingsFile_Sample %>% dplyr::select(Conditions), by= 0)
feature_data <- tibble::column_to_rownames(feature_data, "Row.names")
imputed_feature_data <- feature_data %>%
dplyr::group_by(Conditions) %>%
dplyr::mutate(across(all_of(feature), ~{
if(all(is.na(.))) {
message <- paste0("For some conditions all measured samples are NA for " , feature, ". Hence we can not perform half-minimum value imputation per condition for this metabolite and will assume it is a true biological 0 in those cases.")
logger::log_info(message)
message(message)
return(0) # Return NA if all values are missing
} else {
return(replace(., is.na(.), min(., na.rm = TRUE)*(MVI_Percentage/100)))
}
}))
NA_removed_matrix[[feature]] <- imputed_feature_data[[feature]]
}
if(CoRe==TRUE){
replaceNAdf <- filtered_matrix%>% dplyr::filter(SettingsFile_Sample[[SettingsInfo[["Conditions"]]]] == SettingsInfo[["CoRe_media"]])
# find metabolites with NA
na_percentage <- colMeans(is.na(replaceNAdf)) * 100
highNA_metabs <- na_percentage[na_percentage>20 & na_percentage<100]
OnlyNA_metabs <- na_percentage[na_percentage==100]
# report metabolites with NA
if(sum(na_percentage)>0){
message <- paste0("NA values were found in Control_media samples for metabolites. For metabolites including NAs MVI is performed unless all samples of a metabolite are NA.")
logger::log_info(message)
message(message)
if(sum(na_percentage>20 & na_percentage<100)>0){
message <- paste0("Metabolites with high NA load (>20%) in Control_media samples are: ",paste(names(highNA_metabs), collapse = ", "), ".")
logger::log_info(message)
message(message)
}
if(sum(na_percentage==100)>0){
message <- paste0("Metabolites with only NAs (=100%) in Control_media samples are: ",paste(names(OnlyNA_metabs), collapse = ", "), ". Those NAs are set zero as we consider them true zeros")
logger::log_info(message)
message(message)
}
}
# if all values are NA set to 0
replaceNAdf_zero <- as.data.frame(lapply(replaceNAdf, function(x) if(all(is.na(x))) replace(x, is.na(x), 0) else x))
colnames(replaceNAdf_zero) <- colnames(replaceNAdf)
rownames(replaceNAdf_zero) <- rownames(replaceNAdf)
# If there is at least 1 value use the half minimum per feature
replaceNAdf_Zero_MVI <- apply( replaceNAdf_zero, 2, function(x) {x[is.na(x)] <- min(x, na.rm = TRUE)/2
return(x)
}) %>% as.data.frame()
rownames(replaceNAdf_Zero_MVI) <- rownames(replaceNAdf)
# add the samples in the original dataframe
filtered_matrix_res <- rbind(NA_removed_matrix, replaceNAdf_Zero_MVI)
}else{
filtered_matrix_res <- NA_removed_matrix
}
## ------------------ Return ------------------ ##
invisible(return(filtered_matrix_res))
}
################################################################################################
### ### ### PreProcessing helper function: Total ion Count Normalization ### ### ###
################################################################################################
#' Total ion count normalisazion
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected and consider converting any zeros to NA unless they are true zeros.
#' @param SettingsFile_Sample DF which contains information about the samples, which will be combined with the input data based on the unique sample identifiers used as rownames.
#' @param SettingsInfo Named vector containing the information about the names of the experimental parameters. c(Conditions="ColumnName_Plot_SettingsFile").
#' @param TIC \emph{Optional: } If TRUE, Total Ion Count normalization is performed. If FALSE, only RLA QC plots are returned. \strong{Default = TRUE}
#'
#' @return List with two elements: DF (including output table) and Plot (including all plots generated)
#'
#' @examples
#' Intra <- ToyData("IntraCells_Raw")
#' Res <- TICNorm(InputData=Intra[-c(49:58), -c(1:3)]%>% dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .)),
#' SettingsFile_Sample=Intra[-c(49:58), c(1:3)],
#' SettingsInfo = c(Conditions = "Conditions"))
#'
#' @keywords total ion count normalisation
#'
#' @importFrom magrittr %>% %<>%
#' @importFrom tidyr pivot_longer
#' @importFrom gridExtra grid.arrange
#' @importFrom ggplot2 ggplot geom_boxplot geom_hline labs theme_classic
#' @importFrom ggplot2 theme_minimal theme annotation_custom aes_string element_text
#' @importFrom logger log_info log_trace
#'
#' @noRd
#'
TICNorm <-function(InputData,
SettingsFile_Sample,
SettingsInfo,
TIC=TRUE){
## ------------------ Prepare the data ------------------- ##
NA_removed_matrix <- InputData
NA_removed_matrix[is.na(NA_removed_matrix)] <- 0#replace NA with 0
## ------------------ QC plot ------------------- ##
### Before TIC Normalization
#### Log() transformation:
log_NA_removed_matrix <- suppressWarnings(log(NA_removed_matrix) %>% t() %>% as.data.frame()) # log tranforms the data
nan_count <- sum(is.nan(as.matrix(log_NA_removed_matrix)))# Count NaN values (produced by log(0))
if (nan_count > 0) {# Issue a custom warning if NaNs are present
message <- paste("For the RLA plot before/after TIC normalisation we have to perform log() transformation. This resulted in", nan_count, "NaN values due to 0s in the data.")
logger::log_trace("Warning: ", message, sep="")
warning(message)
}
medians <- apply(log_NA_removed_matrix, 2, median) # get median
RLA_data_raw <- log_NA_removed_matrix - medians # Subtract the medians from each column
RLA_data_long <- tidyr::pivot_longer(RLA_data_raw, cols = everything(), names_to = "Group")
names(RLA_data_long)<- c("Samples", "Intensity")
RLA_data_long <- as.data.frame(RLA_data_long)
for (row in 1:nrow(RLA_data_long)){ # add conditions
RLA_data_long[row, SettingsInfo[["Conditions"]]] <- SettingsFile_Sample[rownames(SettingsFile_Sample) %in%RLA_data_long[row,1],SettingsInfo[["Conditions"]]]
}
# Create the ggplot boxplot
RLA_data_raw <- ggplot2::ggplot(RLA_data_long, ggplot2::aes_string(x = "Samples", y = "Intensity", color = SettingsInfo[["Conditions"]])) +
ggplot2::geom_boxplot() +
ggplot2::geom_hline(yintercept = 0, color = "red", linetype = "solid") +
ggplot2::labs(title = "Before TIC Normalization")+
ggplot2::theme_classic()+
ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1))+
ggplot2::theme(legend.position = "none")
#RLA_data_raw_Sized <- plotGrob_Processing(InputPlot = RLA_data_raw, PlotName= "Before TIC Normalization", PlotType= "RLA")
if(TIC==TRUE){
## ------------------ Perform TIC ------------------- ##
message <- paste0("Total Ion Count (TIC) normalization: Total Ion Count (TIC) normalization is used to reduce the variation from non-biological sources, while maintaining the biological variation. REF: Wulff et. al., (2018), Advances in Bioscience and Biotechnology, 9, 339-351, doi:https://doi.org/10.4236/abb.2018.98022")
logger::log_info(message)
message(message)
RowSums <- rowSums(NA_removed_matrix)
Median_RowSums <- median(RowSums) #This will built the median
Data_TIC_Pre <- apply(NA_removed_matrix, 2, function(i) i/RowSums) #This is dividing the ion intensity by the total ion count
Data_TIC <- Data_TIC_Pre*Median_RowSums #Multiplies with the median metabolite intensity
Data_TIC <- as.data.frame(Data_TIC)
## ------------------ QC plot ------------------- ##
### After TIC normalization
log_Data_TIC <- suppressWarnings(log(Data_TIC) %>% t() %>% as.data.frame()) # log tranforms the data
medians <- apply(log_Data_TIC, 2, median)
RLA_data_norm <- log_Data_TIC - medians # Subtract the medians from each column
RLA_data_long <- tidyr::pivot_longer(RLA_data_norm, cols = everything(), names_to = "Group")
names(RLA_data_long)<- c("Samples", "Intensity")
for (row in 1:nrow(RLA_data_long)){ # add conditions
RLA_data_long[row, SettingsInfo[["Conditions"]]] <- SettingsFile_Sample[rownames(SettingsFile_Sample) %in%RLA_data_long[row,1],SettingsInfo[["Conditions"]]]
}
# Create the ggplot boxplot
RLA_data_norm <- ggplot2::ggplot(RLA_data_long, ggplot2::aes_string(x = "Samples", y = "Intensity", color = SettingsInfo[["Conditions"]])) +
ggplot2::geom_boxplot() +
ggplot2::geom_hline(yintercept = 0, color = "red", linetype = "solid") +
ggplot2::labs(title = "After TIC Normalization")+
ggplot2::theme_classic() +
ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1))+
ggplot2::theme(legend.position = "none")
#RLA_data_norm_Sized <- plotGrob_Processing(InputPlot = RLA_data_norm, PlotName= "After TIC Normalization", PlotType= "RLA")
#Combine Plots
dev.new()
norm_plots <- suppressWarnings(gridExtra::grid.arrange(RLA_data_raw+ ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1))+ ggplot2::theme(legend.position = "none"),
RLA_data_norm+ ggplot2::theme(axis.text.x = ggplot2::element_text(angle = 90, hjust = 1))+ ggplot2::theme(legend.position = "none"),
ncol = 2))
dev.off()
norm_plots <- ggplot2::ggplot() +ggplot2::theme_minimal()+ ggplot2::annotation_custom(norm_plots)
## ------------------ Return ------------------ ##
Output_list <- list("DF" = list("Data_TIC"=as.data.frame(Data_TIC)),"Plot"=list( "norm_plots"=norm_plots, "RLA_AfterTICNorm"=RLA_data_norm, "RLA_BeforeTICNorm" = RLA_data_raw ))
invisible(return(Output_list))
}else{
## ------------------ Return ------------------ ##
Output_list <- list("Plot"=list("RLA_BeforeTICNorm" = RLA_data_raw))
invisible(return(Output_list))
}
}
################################################################################################
### ### ### PreProcessing helper function: CoRe nomalisation ### ### ###
################################################################################################
#' Consumption Release Normalisation
#'
#' @param InputData DF which contains unique sample identifiers as row names and metabolite numerical values in columns with metabolite identifiers as column names. Use NA for metabolites that were not detected and consider converting any zeros to NA unless they are true zeros.
#' @param SettingsFile_Sample DF which contains information about the samples, which will be combined with the input data based on the unique sample identifiers used as rownames.
#' @param SettingsInfo Named vector containing the information about the names of the experimental parameters. c(Conditions="ColumnName_Plot_SettingsFile", CoRe_norm_factor = "Columnname_Input_SettingsFile", CoRe_media = "Columnname_Input_SettingsFile"). Column CoRe_norm_factor is used for normalization and CoRe_media is used to specify the name of the media controls in the Conditions.
#'
#' @return List with two elements: DF (including output table) and Plot (including all plots generated)
#'
#' @examples
#' Media <- ToyData("CultureMedia_Raw")%>% subset(!Conditions=="Pool")%>% dplyr::mutate_all(~ ifelse(grepl("^0*(\\.0*)?$", as.character(.)), NA, .))
#' Res <- CoReNorm(InputData= Media[, -c(1:3)],
#' SettingsFile_Sample= Media[, c(1:3)],
#' SettingsInfo = c(Conditions = "Conditions", CoRe_norm_factor = "GrowthFactor", CoRe_media = "blank"))
#'
#' @keywords Consumption Release Metaqbolomics, Normalisation, Exometabolomics
#'
#' @importFrom magrittr %>% %<>%
#' @importFrom tibble rownames_to_column column_to_rownames
#' @importFrom ggrepel geom_text_repel
#' @importFrom ggplot2 ggplot geom_histogram geom_vline geom_density labs theme_classic geom_violin geom_dotplot
#' @importFrom logger log_info log_trace
#' @importFrom dplyr case_when summarise_all mutate rowwise mutate_all select filter pull
#'
#'
#' @noRd