From 56a403e5feeaa3b47453b43fcf79140d83fe3eaf Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Jan 2023 17:13:27 -0500 Subject: [PATCH 01/53] Switch from snake_case to kebab-case for config params --- config/default.yml | 10 +++++----- config/robust.yml | 10 +++++----- config/test.yml | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/default.yml b/config/default.yml index f22e7b1..839f7b0 100644 --- a/config/default.yml +++ b/config/default.yml @@ -1,11 +1,11 @@ -dataset_csv: data/processed/otu-large.csv -dataset_name: otu-large -outcome_colname: dx -ml_methods: +dataset-csv: data/processed/otu-large.csv +dataset-name: otu-large +outcome-colname: dx +ml-methods: - glmnet - rf kfold: 5 ncores: 8 nseeds: 10 -find_feature_importance: true +find-feature-importance: true hyperparams: \ No newline at end of file diff --git a/config/robust.yml b/config/robust.yml index d59d4db..c17768a 100644 --- a/config/robust.yml +++ b/config/robust.yml @@ -1,7 +1,7 @@ -dataset_csv: data/processed/otu-large.csv -dataset_name: otu-large -outcome_colname: dx -ml_methods: +dataset-csv: data/processed/otu-large.csv +dataset-name: otu-large +outcome-colname: dx +ml-methods: - glmnet - rf - rpart2 @@ -9,7 +9,7 @@ ml_methods: kfold: 5 ncores: 36 nseeds: 100 -find_feature_importance: false +find-feature-importance: false hyperparams: - glmnet: - alpha: diff --git a/config/test.yml b/config/test.yml index 42cef2a..61e757f 100644 --- a/config/test.yml +++ b/config/test.yml @@ -1,12 +1,12 @@ -dataset_csv: data/processed/otu-mini-bin.csv -dataset_name: otu-mini-bin -outcome_colname: dx -ml_methods: +dataset-csv: data/processed/otu-mini-bin.csv +dataset-name: otu-mini-bin +outcome-colname: dx +ml-methods: - glmnet kfold: 2 ncores: 4 nseeds: 2 -find_feature_importance: true +find-feature-importance: true hyperparams: - glmnet: - alpha: From 754329d944c4cb466c700a30182370c2cb6137dd Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 18 Jan 2023 17:13:45 -0500 Subject: [PATCH 02/53] Quick proof of concept for paramspace (#36) --- workflow/Snakefile | 20 ++++++++++++++------ workflow/envs/smk.yml | 1 + workflow/rules/learn.smk | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index c118665..f609f1a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,24 +1,32 @@ import os - +import pandas as pd +from snakemake.utils import Paramspace configfile: "config/default.yml" - MEM_PER_GB = 1024 -dataset = config["dataset_name"] +dataset_filename = config['dataset-csv'] +dataset = config["dataset-name"] ncores = config["ncores"] -ml_methods = config["ml_methods"] +ml_methods = config["ml-methods"] kfold = config["kfold"] -outcome_colname = config["outcome_colname"] +outcome_colname = config["outcome-colname"] nseeds = config["nseeds"] start_seed = 100 seeds = range(start_seed, start_seed + nseeds) hyperparams = config["hyperparams"] if "hyperparams" in config else None -find_feature_importance = config["find_feature_importance"] +find_feature_importance = config["find-feature-importance"] + +ignore_keys = ['dataset-csv', 'outcome-colname', 'hyperparams', 'find-feature-importance', 'nseeds', 'ncores'] +for k in ignore_keys: + config.pop(k) +paramspace = Paramspace(pd.DataFrame.from_dict(config), param_sep = "_") +print('paramspace.wildcard_pattern:\t', paramspace.wildcard_pattern) +print('paramspace.instance_patterns:\t', [i for i in paramspace.instance_patterns]) include: "rules/learn.smk" include: "rules/combine.smk" diff --git a/workflow/envs/smk.yml b/workflow/envs/smk.yml index 8461f70..a462104 100644 --- a/workflow/envs/smk.yml +++ b/workflow/envs/smk.yml @@ -4,4 +4,5 @@ channels: - bioconda dependencies: - graphviz + - pandas - snakemake=7 \ No newline at end of file diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index f39ebda..d681785 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -1,7 +1,7 @@ rule preprocess_data: input: R="workflow/scripts/preproc.R", - csv=config["dataset_csv"], + csv=dataset_filename, output: rds=f"data/processed/{dataset}_preproc.Rds", log: From 00346d7a36732c812d3ddd0a19820983a72d6335 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 20 Jan 2023 17:00:25 -0500 Subject: [PATCH 03/53] Config key names need to be valid Python Can't have hyphens. Better to use hyphens to separate params. --- config/default.yml | 10 +++++----- config/robust.yml | 10 +++++----- config/test.yml | 10 +++++----- data/processed/{otu-large.csv => otu_large.csv} | 0 .../{otu-mini-bin.csv => otu_mini_bin.csv} | 0 workflow/Snakefile | 17 +++++++++-------- 6 files changed, 24 insertions(+), 23 deletions(-) rename data/processed/{otu-large.csv => otu_large.csv} (100%) rename data/processed/{otu-mini-bin.csv => otu_mini_bin.csv} (100%) diff --git a/config/default.yml b/config/default.yml index 839f7b0..7ed91ea 100644 --- a/config/default.yml +++ b/config/default.yml @@ -1,11 +1,11 @@ -dataset-csv: data/processed/otu-large.csv -dataset-name: otu-large -outcome-colname: dx -ml-methods: +dataset_csv: data/processed/otu_large.csv +dataset_name: otu_large +outcome_colname: dx +ml_methods: - glmnet - rf kfold: 5 ncores: 8 nseeds: 10 -find-feature-importance: true +find_feature_importance: true hyperparams: \ No newline at end of file diff --git a/config/robust.yml b/config/robust.yml index c17768a..804f79e 100644 --- a/config/robust.yml +++ b/config/robust.yml @@ -1,7 +1,7 @@ -dataset-csv: data/processed/otu-large.csv -dataset-name: otu-large -outcome-colname: dx -ml-methods: +dataset_csv: data/processed/otu_large.csv +dataset_name: otu_large +outcome_colname: dx +ml_methods: - glmnet - rf - rpart2 @@ -9,7 +9,7 @@ ml-methods: kfold: 5 ncores: 36 nseeds: 100 -find-feature-importance: false +find_feature_importance: false hyperparams: - glmnet: - alpha: diff --git a/config/test.yml b/config/test.yml index 61e757f..8baa8db 100644 --- a/config/test.yml +++ b/config/test.yml @@ -1,12 +1,12 @@ -dataset-csv: data/processed/otu-mini-bin.csv -dataset-name: otu-mini-bin -outcome-colname: dx -ml-methods: +dataset_csv: data/processed/otu_mini_bin.csv +dataset_name: otu_mini_bin +outcome_colname: dx +ml_methods: - glmnet kfold: 2 ncores: 4 nseeds: 2 -find-feature-importance: true +find_feature_importance: true hyperparams: - glmnet: - alpha: diff --git a/data/processed/otu-large.csv b/data/processed/otu_large.csv similarity index 100% rename from data/processed/otu-large.csv rename to data/processed/otu_large.csv diff --git a/data/processed/otu-mini-bin.csv b/data/processed/otu_mini_bin.csv similarity index 100% rename from data/processed/otu-mini-bin.csv rename to data/processed/otu_mini_bin.csv diff --git a/workflow/Snakefile b/workflow/Snakefile index f609f1a..db53868 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -6,25 +6,26 @@ configfile: "config/default.yml" MEM_PER_GB = 1024 -dataset_filename = config['dataset-csv'] -dataset = config["dataset-name"] +dataset_filename = config['dataset_csv'] +dataset = config["dataset_name"] ncores = config["ncores"] -ml_methods = config["ml-methods"] +ml_methods = config["ml_methods"] kfold = config["kfold"] -outcome_colname = config["outcome-colname"] +outcome_colname = config["outcome_colname"] nseeds = config["nseeds"] start_seed = 100 seeds = range(start_seed, start_seed + nseeds) hyperparams = config["hyperparams"] if "hyperparams" in config else None -find_feature_importance = config["find-feature-importance"] +find_feature_importance = config["find_feature_importance"] -ignore_keys = ['dataset-csv', 'outcome-colname', 'hyperparams', 'find-feature-importance', 'nseeds', 'ncores'] +ignore_keys = ['dataset_csv', 'ncores', 'nseeds', 'find_feature_importance', 'hyperparams'] for k in ignore_keys: - config.pop(k) + config.pop(k, None) +config['seed'] = list(seeds) -paramspace = Paramspace(pd.DataFrame.from_dict(config), param_sep = "_") +paramspace = Paramspace(pd.DataFrame.from_dict(config), param_sep = "-") print('paramspace.wildcard_pattern:\t', paramspace.wildcard_pattern) print('paramspace.instance_patterns:\t', [i for i in paramspace.instance_patterns]) From 8461a3cc0339d51e303f88be37f424f246be8208 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 20 Jan 2023 17:01:50 -0500 Subject: [PATCH 04/53] WIP: test paramspace.instance in R & permuations --- workflow/rules/combine.smk | 11 +++++------ workflow/rules/learn.smk | 29 ++++++++++++++++++++--------- workflow/scripts/test_paramspace.R | 4 ++++ workflow/scripts/test_paramspace.py | 23 +++++++++++++++++++++++ 4 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 workflow/scripts/test_paramspace.R create mode 100644 workflow/scripts/test_paramspace.py diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 2a072af..8e8d633 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -3,16 +3,15 @@ rule combine_results: input: R="workflow/scripts/combine_results.R", csv=expand( - "results/{{dataset}}/runs/{method}_{seed}_{{type}}.csv", - method=ml_methods, - seed=seeds, + "results/{params}/{{type}}.csv", + params=paramspace.instance_patterns ), output: - csv="results/{dataset}/{type}_results.csv", + csv="results/{type}_results.csv", log: - "log/{dataset}/combine_results_{type}.txt", + "log/combine_results_{type}.txt", benchmark: - "benchmarks/{dataset}/combine_results_{type}.txt" + "benchmarks/combine_results_{type}.txt" conda: "../envs/mikropml.yml" script: diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index d681785..494d594 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -24,18 +24,15 @@ rule run_ml: R="workflow/scripts/train_ml.R", rds=rules.preprocess_data.output.rds, output: - model="results/{dataset}/runs/{method}_{seed}_model.Rds", - perf="results/{dataset}/runs/{method}_{seed}_performance.csv", - test="results/{dataset}/runs/{method}_{seed}_test-data.csv", + model=f"results/{paramspace.wildcard_pattern}/model.Rds", + perf=f"results/{paramspace.wildcard_pattern}/performance.csv", + test=f"results/{paramspace.wildcard_pattern}/test-data.csv", log: - "log/{dataset}/runs/run_ml.{method}_{seed}.txt", + f"log/{paramspace.wildcard_pattern}/run_ml.txt", benchmark: - "benchmarks/{dataset}/runs/run_ml.{method}_{seed}.txt" + f"benchmarks/{paramspace.wildcard_pattern}/run_ml.txt" params: - outcome_colname=outcome_colname, - method="{method}", - seed="{seed}", - kfold=kfold, + params=paramspace.instance, hyperparams=hyperparams, threads: ncores resources: @@ -45,6 +42,20 @@ rule run_ml: script: "../scripts/train_ml.R" +rule test_paramspace: + input: + csv=dataset_filename + output: + rds=f"tmp/{paramspace.wildcard_pattern}/test.Rds" + params: + params=paramspace.instance, + log: f"log/{paramspace.wildcard_pattern}/test_paramspace.txt" + script: + "../scripts/test_paramspace.R" + +rule agg_params: + input: + expand("tmp/{params}/test.Rds", params = paramspace.instance_patterns) rule find_feature_importance: input: diff --git a/workflow/scripts/test_paramspace.R b/workflow/scripts/test_paramspace.R new file mode 100644 index 0000000..a613aad --- /dev/null +++ b/workflow/scripts/test_paramspace.R @@ -0,0 +1,4 @@ +schtools::log_snakemake() +print(snakemake@params[['params']]) + +saveRDS(snakemake@params[['params']], snakemake@output[['rds']]) diff --git a/workflow/scripts/test_paramspace.py b/workflow/scripts/test_paramspace.py new file mode 100644 index 0000000..01e696f --- /dev/null +++ b/workflow/scripts/test_paramspace.py @@ -0,0 +1,23 @@ +from itertools import product +import pandas as pd +from snakemake.utils import Paramspace +import yaml + +with open('config/robust.yml', 'r') as infile: + config = yaml.load(infile, Loader=yaml.Loader) + +ignore_keys = ['dataset_csv', 'outcome_colname', 'hyperparams', 'find_feature_importance', 'ncores', 'nseeds'] +for k in ignore_keys: + config.pop(k, None) + +config['seed'] = list(range(100, 102)) +conf_lists = {k:v for k,v in config.items() if type(v) == list} +params_df = pd.DataFrame(list(product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) +for k in conf_lists.keys(): + config.pop(k) +for k, v in config.items(): + params_df[k] = v + +paramspace = Paramspace(params_df, param_sep = "_") +print('paramspace.wildcard_pattern:\t', paramspace.wildcard_pattern) +print('paramspace.instance_patterns:\t', [i for i in paramspace.instance_patterns]) \ No newline at end of file From 63dd26c9cbef4a0b091e89a3ac9ffeb47618c2e7 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Wed, 25 Jan 2023 14:33:23 -0500 Subject: [PATCH 05/53] Get dot product of config lists for paramspace --- workflow/Snakefile | 14 ++++++++++++-- workflow/rules/combine.smk | 17 ++++++++--------- workflow/rules/learn.smk | 6 +++--- workflow/rules/plot.smk | 28 ++++++++++++++-------------- 4 files changed, 37 insertions(+), 28 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index db53868..55a5869 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,4 +1,5 @@ import os +import itertools as it import pandas as pd from snakemake.utils import Paramspace @@ -24,11 +25,20 @@ ignore_keys = ['dataset_csv', 'ncores', 'nseeds', 'find_feature_importance', 'hy for k in ignore_keys: config.pop(k, None) config['seed'] = list(seeds) +conf_lists = {k:v for k,v in config.items() if type(v) == list} +params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) +for k in conf_lists.keys(): + config.pop(k) +for k, v in config.items(): + params_df[k] = v -paramspace = Paramspace(pd.DataFrame.from_dict(config), param_sep = "-") +paramspace = Paramspace(params_df, param_sep = "-") print('paramspace.wildcard_pattern:\t', paramspace.wildcard_pattern) print('paramspace.instance_patterns:\t', [i for i in paramspace.instance_patterns]) +wildcard_constraints: + kfold = '[0-9]+' + include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" @@ -45,7 +55,7 @@ rule render_report: R="workflow/scripts/render.R", Rmd="report.Rmd", perf_plot=rules.plot_performance.output.plot, - feat_plot="figures/{dataset}/feature_importance.png", + feat_plot=rules.plot_feature_importance.output.plot, hp_plot=expand( "figures/{{dataset}}/hp_performance_{method}.png", method=ml_methods ), diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 8e8d633..f926fa6 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -7,7 +7,7 @@ rule combine_results: params=paramspace.instance_patterns ), output: - csv="results/{type}_results.csv", + csv="results/{type}-results.csv", log: "log/combine_results_{type}.txt", benchmark: @@ -23,11 +23,11 @@ rule combine_hp_performance: R="workflow/scripts/combine_hp_perf.R", rds=expand("results/{{dataset}}/runs/{{method}}_{seed}_model.Rds", seed=seeds), output: - rds="results/{dataset}/hp_performance_results_{method}.Rds", + rds="results/hp_performance_results_{method}.Rds", log: - "log/{dataset}/combine_hp_perf_{method}.txt", + "log/combine_hp_perf_{method}.txt", benchmark: - "benchmarks/{dataset}/combine_hp_perf_{method}.txt" + "benchmarks/combine_hp_perf_{method}.txt" resources: mem_mb=MEM_PER_GB * 16, conda: @@ -40,14 +40,13 @@ rule combine_benchmarks: input: R="workflow/scripts/combine_benchmarks.R", tsv=expand( - "benchmarks/{{dataset}}/runs/run_ml.{method}_{seed}.txt", - method=ml_methods, - seed=seeds, + "benchmarks/{params}/run_ml.txt", + params=paramspace.instance_patterns ), output: - csv="results/{dataset}/benchmarks_results.csv", + csv="results/benchmarks_results.csv", log: - "log/{dataset}/combine_benchmarks.txt", + "log/combine_benchmarks.txt", conda: "../envs/mikropml.yml" script: diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 494d594..1aae1fe 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -26,7 +26,7 @@ rule run_ml: output: model=f"results/{paramspace.wildcard_pattern}/model.Rds", perf=f"results/{paramspace.wildcard_pattern}/performance.csv", - test=f"results/{paramspace.wildcard_pattern}/test-data.csv", + test=f"results/{paramspace.wildcard_pattern}/test_data.csv", log: f"log/{paramspace.wildcard_pattern}/run_ml.txt", benchmark: @@ -63,9 +63,9 @@ rule find_feature_importance: model=rules.run_ml.output.model, test=rules.run_ml.output.test, output: - feat="results/{dataset}/runs/{method}_{seed}_feature-importance.csv", + feat=f"results/{paramspace.wildcard_pattern}/feature_importance.csv", log: - "log/{dataset}/runs/find_feature-importance.{method}_{seed}.txt", + f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", params: outcome_colname=outcome_colname, method="{method}", diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 38a118a..46575af 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -1,11 +1,11 @@ rule plot_performance: input: R="workflow/scripts/plot_performance.R", - csv="results/{dataset}/performance_results.csv", + csv="results/performance-results.csv", output: - plot="figures/{dataset}/performance.png", + plot="figures/performance.png", log: - "log/{dataset}/plot_performance.txt", + "log/plot_performance.txt", conda: "../envs/mikropml.yml" script: @@ -17,11 +17,11 @@ if find_feature_importance: rule plot_feature_importance: input: R="workflow/scripts/plot_feature_importance.R", - csv="results/{dataset}/feature-importance_results.csv", + csv="results/feature_importance-results.csv", output: - plot="figures/{dataset}/feature_importance.png", + plot="figures/feature_importance.png", log: - "log/{dataset}/plot_feature_importance.txt", + "log/plot_feature_importance.txt", conda: "../envs/mikropml.yml" script: @@ -32,9 +32,9 @@ else: rule make_blank_feature_plot: output: - plot="figures/{dataset}/feature_importance.png", + plot="figures/feature_importance.png", log: - "log/{dataset}/make_blank_plot.txt", + "log/make_blank_plot.txt", conda: "../envs/mikropml.yml" script: @@ -44,11 +44,11 @@ else: rule plot_hp_performance: input: R="workflow/scripts/plot_hp_perf.R", - rds="results/{dataset}/hp_performance_results_{method}.Rds", + rds="results/hp_performance_results_{method}.Rds", output: - plot="figures/{dataset}/hp_performance_{method}.png", + plot="figures/hp_performance_{method}.png", log: - "log/{dataset}/plot_hp_perf_{method}.txt", + "log/plot_hp_perf_{method}.txt", conda: "../envs/mikropml.yml" script: @@ -58,11 +58,11 @@ rule plot_hp_performance: rule plot_benchmarks: input: R="workflow/scripts/plot_benchmarks.R", - csv="results/{dataset}/benchmarks_results.csv", + csv="results/benchmarks-results.csv", output: - plot="figures/{dataset}/benchmarks.png", + plot="figures/benchmarks.png", log: - "log/{dataset}/plot_benchmarks.txt", + "log/plot_benchmarks.txt", conda: "../envs/mikropml.yml" script: From 6306e352529da23e0dab655e885cf7dea25f4b34 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 26 Jan 2023 01:20:20 -0500 Subject: [PATCH 06/53] Fix wildcards --- config/config.yaml | 4 ++-- config/robust.yaml | 4 ++-- config/test.yaml | 4 ++-- workflow/Snakefile | 28 ++++++++++++++++------------ workflow/rules/combine.smk | 6 +++--- workflow/rules/example-report.smk | 2 +- workflow/rules/learn.smk | 10 +++++----- workflow/rules/plot.smk | 4 ++-- 8 files changed, 33 insertions(+), 29 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 6bd05a7..faa837a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ -dataset_name: otu_large +dataset: otu_large outcome_colname: dx -ml_methods: +ml_method: - glmnet - rf kfold: 5 diff --git a/config/robust.yaml b/config/robust.yaml index 2d072fd..c72e55c 100644 --- a/config/robust.yaml +++ b/config/robust.yaml @@ -1,6 +1,6 @@ -dataset_name: otu_large +dataset: otu_large outcome_colname: dx -ml_methods: +ml_method: - glmnet - rf - rpart2 diff --git a/config/test.yaml b/config/test.yaml index 6860b2c..c13d675 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -1,6 +1,6 @@ -dataset_name: otu_micro +dataset: otu_micro outcome_colname: dx -ml_methods: +ml_method: - glmnet kfold: 2 ncores: 4 diff --git a/workflow/Snakefile b/workflow/Snakefile index fc0ebc1..59d91aa 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,9 +27,9 @@ config_path = ( MEM_PER_GB = 1024 -dataset = config["dataset_name"] +dataset = config["dataset"] ncores = config["ncores"] -ml_methods = config["ml_methods"] +ml_methods = config["ml_method"] kfold = config["kfold"] outcome_colname = config["outcome_colname"] @@ -40,7 +40,10 @@ seeds = range(start_seed, start_seed + nseeds) hyperparams = config["hyperparams"] if "hyperparams" in config else None find_feature_importance = config["find_feature_importance"] -ignore_keys = ['dataset_csv', 'ncores', 'nseeds', 'find_feature_importance', 'hyperparams'] +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Prepare Parameter Space +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +ignore_keys = ['dataset_csv', 'outcome_colname', 'ncores', 'nseeds', 'find_feature_importance', 'hyperparams'] for k in ignore_keys: config.pop(k, None) config['seed'] = list(seeds) @@ -51,10 +54,12 @@ for k in conf_lists.keys(): for k, v in config.items(): params_df[k] = v +params_df = params_df[sorted(params_df.columns.tolist())] paramspace = Paramspace(params_df, param_sep = "-") paramspace_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) paramspace_no_seed = re.sub("/seed-{seed}/", "/", paramspace.wildcard_pattern) print(paramspace.wildcard_pattern) +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # wildcard_constraints: kfold = '[0-9]+' @@ -62,7 +67,7 @@ wildcard_constraints: include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" -include: "rules/example-report.smk" +#include: "rules/example-report.smk" report: "report/workflow.rst" @@ -70,16 +75,16 @@ report: "report/workflow.rst" rule targets: input: - f"workflow_{dataset}.zip", + f"workflow-{dataset}.zip", rule render_report: input: perf_plot=rules.plot_performance.output.plot, feat_plot=rules.plot_feature_importance.output.plot, - hp_plot=expand( - rules.plot_hp_performance.output.plot, ml_method=ml_methods - ), + # hp_plot=expand( + # rules.plot_hp_performance.output.plot, ml_method=ml_methods + # ), bench_plot=rules.plot_benchmarks.output.plot, roc_plot=rules.plot_roc_curves.output.plot, rulegraph="figures/graphviz/rulegraph.png", @@ -106,12 +111,11 @@ rule archive: rules.render_report.input, rules.render_report.output, expand( - "results/{dataset}/{rtype}_results.csv", - dataset=dataset, - rtype=["performance", "feature-importance", "benchmarks", "sensspec"], + "results/{rtype}-results.csv", + rtype=["performance", "feature_importance", "benchmarks", "sensspec"], ), output: - "workflow_{dataset}.zip", + "workflow-{dataset}.zip", log: "log/archive_{dataset}.txt", conda: diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 1ab44f8..08f66e8 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -21,11 +21,11 @@ rule combine_hp_performance: input: rds=expand(f"results/{paramspace_tame_seed}/model.Rds", seed=seeds), output: - rds="results/ml_methods-{ml_methods}/hp_performance_results.Rds", + rds=f"results/{paramspace_no_seed}/hp_performance_results.Rds", log: - "log/ml_methods-{ml_methods}/combine_hp_perf.txt", + f"log/{paramspace_no_seed}/combine_hp_perf.txt", benchmark: - "benchmarks/ml_methods-{ml_methods}/combine_hp_perf.txt" + f"benchmarks/{paramspace_no_seed}/combine_hp_perf.txt" resources: mem_mb=MEM_PER_GB * 16, conda: diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index fea95bb..a5ead0a 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -14,7 +14,7 @@ rule copy_example_figures: feat_plot="figures/example/feature_importance.png", bench_plot="figures/example/benchmarks.png", hp_plot=expand("figures/example/hp_performance_{ml_method}.png", - method=ml_methods), + ml_method=ml_methods), roc_plot="figures/example/roc_curves.png", rulegraph="figures/example/rulegraph.png", log: diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 82cc08f..a8f3d13 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -1,12 +1,12 @@ rule preprocess_data: input: - csv="data/processed/{datasest}.csv", + csv=f"data/processed/{dataset}.csv", output: - rds="data/processed/{dataset}_preproc.Rds", + rds=f"data/processed/{dataset}_preproc.Rds", log: - "log/{dataset}/preprocess_data.txt", + f"log/{dataset}/preprocess_data.txt", benchmark: - "benchmarks/{dataset}/preprocess_data.txt" + f"benchmarks/{dataset}/preprocess_data.txt" params: outcome_colname=outcome_colname, threads: ncores @@ -50,7 +50,7 @@ rule find_feature_importance: f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", params: outcome_colname=outcome_colname, - method="{method}", + method="{ml_method}", seed="{seed}", threads: ncores resources: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 1f12744..8e9deb4 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -47,9 +47,9 @@ else: "../scripts/make_blank_plot.R" -rule plot_hp_performance: +rule plot_hp_performance: # TODO: modify this to take list of all hp results (without being precombined) and make a big facet_wrap or plot_grid input: - rds="results/ml_methods-{ml_method}/hp_performance_results.Rds", + rds=f"results/{paramspace_no_seed}/hp_performance_results.Rds", output: plot=report( "figures/hp_performance_{ml_method}.png", From 63249bb334dd0aab06135b46df50ed68397552d4 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Fri, 27 Jan 2023 23:53:28 -0500 Subject: [PATCH 07/53] Remove old test code --- workflow/scripts/test_paramspace.R | 4 ---- workflow/scripts/test_paramspace.py | 23 ----------------------- 2 files changed, 27 deletions(-) delete mode 100644 workflow/scripts/test_paramspace.R delete mode 100644 workflow/scripts/test_paramspace.py diff --git a/workflow/scripts/test_paramspace.R b/workflow/scripts/test_paramspace.R deleted file mode 100644 index a613aad..0000000 --- a/workflow/scripts/test_paramspace.R +++ /dev/null @@ -1,4 +0,0 @@ -schtools::log_snakemake() -print(snakemake@params[['params']]) - -saveRDS(snakemake@params[['params']], snakemake@output[['rds']]) diff --git a/workflow/scripts/test_paramspace.py b/workflow/scripts/test_paramspace.py deleted file mode 100644 index 01e696f..0000000 --- a/workflow/scripts/test_paramspace.py +++ /dev/null @@ -1,23 +0,0 @@ -from itertools import product -import pandas as pd -from snakemake.utils import Paramspace -import yaml - -with open('config/robust.yml', 'r') as infile: - config = yaml.load(infile, Loader=yaml.Loader) - -ignore_keys = ['dataset_csv', 'outcome_colname', 'hyperparams', 'find_feature_importance', 'ncores', 'nseeds'] -for k in ignore_keys: - config.pop(k, None) - -config['seed'] = list(range(100, 102)) -conf_lists = {k:v for k,v in config.items() if type(v) == list} -params_df = pd.DataFrame(list(product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) -for k in conf_lists.keys(): - config.pop(k) -for k, v in config.items(): - params_df[k] = v - -paramspace = Paramspace(params_df, param_sep = "_") -print('paramspace.wildcard_pattern:\t', paramspace.wildcard_pattern) -print('paramspace.instance_patterns:\t', [i for i in paramspace.instance_patterns]) \ No newline at end of file From 279e7ddc5d0e70cfd8b072e41332b91efe378b5b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 00:03:14 -0500 Subject: [PATCH 08/53] Move un-pre-processed data to 'data/' --- data/{processed => }/otu_large.csv | 0 data/{processed => }/otu_micro.csv | 0 data/{processed => }/otu_mini_bin.csv | 0 workflow/rules/learn.smk | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename data/{processed => }/otu_large.csv (100%) rename data/{processed => }/otu_micro.csv (100%) rename data/{processed => }/otu_mini_bin.csv (100%) diff --git a/data/processed/otu_large.csv b/data/otu_large.csv similarity index 100% rename from data/processed/otu_large.csv rename to data/otu_large.csv diff --git a/data/processed/otu_micro.csv b/data/otu_micro.csv similarity index 100% rename from data/processed/otu_micro.csv rename to data/otu_micro.csv diff --git a/data/processed/otu_mini_bin.csv b/data/otu_mini_bin.csv similarity index 100% rename from data/processed/otu_mini_bin.csv rename to data/otu_mini_bin.csv diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index a8f3d13..f04295d 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -1,6 +1,6 @@ rule preprocess_data: input: - csv=f"data/processed/{dataset}.csv", + csv=f"data/{dataset}.csv", output: rds=f"data/processed/{dataset}_preproc.Rds", log: From 8561b1cb96a96c6ff50b1948fa1249611ba33390 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 00:11:46 -0500 Subject: [PATCH 09/53] Implement 'exclude_param_keys' in config to give users control over paramspace wildcards --- config/config.yaml | 9 ++++++++- config/robust.yaml | 7 +++++++ config/test.yaml | 7 +++++++ workflow/Snakefile | 18 ++++++++++-------- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index faa837a..e8185a7 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,4 +7,11 @@ kfold: 5 ncores: 8 nseeds: 10 find_feature_importance: true -hyperparams: \ No newline at end of file +hyperparams: +exclude_param_keys: + - exclude_param_keys + - outcome_colname + - ncores + - nseeds + - find_feature_importance + - hyperparams diff --git a/config/robust.yaml b/config/robust.yaml index c72e55c..f4b3998 100644 --- a/config/robust.yaml +++ b/config/robust.yaml @@ -25,3 +25,10 @@ hyperparams: - 42 - 83 - 166 +exclude_param_keys: + - exclude_param_keys + - outcome_colname + - ncores + - nseeds + - find_feature_importance + - hyperparams diff --git a/config/test.yaml b/config/test.yaml index c13d675..125a827 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -17,3 +17,10 @@ hyperparams: - 0.1 - 1 - 10 +exclude_param_keys: + - exclude_param_keys + - outcome_colname + - ncores + - nseeds + - find_feature_importance + - hyperparams diff --git a/workflow/Snakefile b/workflow/Snakefile index 59d91aa..1d270bf 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -29,22 +29,22 @@ MEM_PER_GB = 1024 dataset = config["dataset"] ncores = config["ncores"] -ml_methods = config["ml_method"] -kfold = config["kfold"] -outcome_colname = config["outcome_colname"] +ml_methods = config["ml_method"] if 'ml_method' in config else 'glmnet' +kfold = config["kfold"] if "kfold" in config else 5 +outcome_colname = config["outcome_colname"] if "outcome_colname" in config else None -nseeds = config["nseeds"] +nseeds = config["nseeds"] if 'nseeds' in config else 1 start_seed = 100 seeds = range(start_seed, start_seed + nseeds) hyperparams = config["hyperparams"] if "hyperparams" in config else None -find_feature_importance = config["find_feature_importance"] +find_feature_importance = config["find_feature_importance"] if "find_feature_importance" in config else None # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # Prepare Parameter Space # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -ignore_keys = ['dataset_csv', 'outcome_colname', 'ncores', 'nseeds', 'find_feature_importance', 'hyperparams'] -for k in ignore_keys: +exclude_param_keys = config['exclude_param_keys'] +for k in exclude_param_keys: config.pop(k, None) config['seed'] = list(seeds) conf_lists = {k:v for k,v in config.items() if type(v) == list} @@ -56,9 +56,11 @@ for k, v in config.items(): params_df = params_df[sorted(params_df.columns.tolist())] paramspace = Paramspace(params_df, param_sep = "-") + +print("Wildcard pattern:", paramspace.wildcard_pattern) + paramspace_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) paramspace_no_seed = re.sub("/seed-{seed}/", "/", paramspace.wildcard_pattern) -print(paramspace.wildcard_pattern) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # wildcard_constraints: From 6110b0f5e84168ef7196c09e31378493dc9c07f2 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 00:13:00 -0500 Subject: [PATCH 10/53] Doc 'exclude_param_keys' & hardwrap lines --- config/README.md | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/config/README.md b/config/README.md index 1f65628..185b1a6 100644 --- a/config/README.md +++ b/config/README.md @@ -1,22 +1,39 @@ # General configuration -To configure this workflow, modify [`config/config.yaml`](/config/config.yaml) according to your needs. +To configure this workflow, modify [`config/config.yaml`](/config/config.yaml) +according to your needs. **Configuration options:** - - `dataset_csv`: the path to the dataset as a csv file. - - `dataset_name`: a short name to identify the dataset. - - `outcome_colname`: column name of the outcomes or classes for the dataset. If blank, the first column of the dataset will be used as the outcome and all other columns are features. - - `ml_methods`: list of machine learning methods to use. Must be [supported by mikropml or caret](http://www.schlosslab.org/mikropml/articles/introduction.html#the-methods-we-support). + - `dataset`: a short name to identify the dataset. The csv file for your + dataset is assumed to be located at `data/{dataset}.csv`. + The dataset should contain one outcome column with all other columns as + features for machine learning. + - `outcome_colname`: column name of the outcomes or classes for the dataset. + If blank, the first column of the dataset will be used as the outcome and + all other columns are features. + - `ml_methods`: list of machine learning methods to use. Must be + [supported by mikropml or caret](http://www.schlosslab.org/mikropml/articles/introduction.html#the-methods-we-support). - `kfold`: k number for k-fold cross validation during model training. - - `ncores`: the number of cores to use for `preprocess_data()`, `run_ml()`, and `get_feature_importance()`. Do not exceed the number of cores you have available. - - `nseeds`: the number of different random seeds to use for training models with `run_ml()`. This will result in `nseeds` different train/test splits of the dataset. - - `find_feature_importance`: whether to calculate feature importances with permutation tests (`true` or `false`). If `false`, the plot in the report will be blank. - - `hyperparams`: override the default model hyperparameters set by mikropml for each ML method (optional). Leave this blank if you'd like to use the defaults. You will have to set these if you wish to use an ML method from caret that we don't officially support. - -We also provide [`config/test.yaml`](/config/test.yaml), which uses a smaller dataset so -you can first make sure the workflow runs without error on your machine -before using your own dataset and custom parameters. + - `ncores`: the number of cores to use for `preprocess_data()`, `run_ml()`, + and `get_feature_importance()`. Do not exceed the number of cores you have available. + - `nseeds`: the number of different random seeds to use for training models + with `run_ml()`. This will result in `nseeds` different train/test splits + of the dataset. + - `find_feature_importance`: whether to calculate feature importances with + permutation tests (`true` or `false`). If `false`, the plot in the report + will be blank. + - `hyperparams`: override the default model hyperparameters set by mikropml + for each ML method (optional). Leave this blank if you'd like to use the + defaults. You will have to set these if you wish to use an ML method from + caret that we don't officially support. + - `exclude_param_keys`: keys in the configfile to exclude from the parameter + space. All keys in the configfile not listed in `exclude_param_keys` will be + included as wildcards for `run_ml` and other rules. + +We also provide [`config/test.yaml`](/config/test.yaml), which uses a smaller +dataset so you can first make sure the workflow runs without error on your +machine before using your own large dataset and custom parameters. The default and test config files are suitable for initial testing, but we recommend using more cores (if available) and From 3436877daf3d4da905adf277d9de15b527b90291 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 00:15:55 -0500 Subject: [PATCH 11/53] Move config & paramspace to separate rule/modules --- workflow/Snakefile | 50 +++-------------------------------- workflow/rules/config.smk | 21 +++++++++++++++ workflow/rules/paramspace.smk | 23 ++++++++++++++++ 3 files changed, 48 insertions(+), 46 deletions(-) create mode 100644 workflow/rules/config.smk create mode 100644 workflow/rules/paramspace.smk diff --git a/workflow/Snakefile b/workflow/Snakefile index 1d270bf..c067d23 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -14,58 +14,16 @@ containerized: "docker://kellysovacool/mikropml:latest" default_configfile = "config/config.yaml" - -configfile: default_configfile - - -args = sys.argv -config_path = ( - args[args.index("--configfile") + 1] - if "--configfile" in args - else default_configfile -) - MEM_PER_GB = 1024 -dataset = config["dataset"] -ncores = config["ncores"] -ml_methods = config["ml_method"] if 'ml_method' in config else 'glmnet' -kfold = config["kfold"] if "kfold" in config else 5 -outcome_colname = config["outcome_colname"] if "outcome_colname" in config else None - -nseeds = config["nseeds"] if 'nseeds' in config else 1 -start_seed = 100 -seeds = range(start_seed, start_seed + nseeds) - -hyperparams = config["hyperparams"] if "hyperparams" in config else None -find_feature_importance = config["find_feature_importance"] if "find_feature_importance" in config else None - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# Prepare Parameter Space -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -exclude_param_keys = config['exclude_param_keys'] -for k in exclude_param_keys: - config.pop(k, None) -config['seed'] = list(seeds) -conf_lists = {k:v for k,v in config.items() if type(v) == list} -params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) -for k in conf_lists.keys(): - config.pop(k) -for k, v in config.items(): - params_df[k] = v - -params_df = params_df[sorted(params_df.columns.tolist())] -paramspace = Paramspace(params_df, param_sep = "-") - -print("Wildcard pattern:", paramspace.wildcard_pattern) - -paramspace_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) -paramspace_no_seed = re.sub("/seed-{seed}/", "/", paramspace.wildcard_pattern) -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +configfile: default_configfile wildcard_constraints: kfold = '[0-9]+' +include: 'rules/config.smk' +include: 'rules/paramspace.smk' + include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk new file mode 100644 index 0000000..d334b54 --- /dev/null +++ b/workflow/rules/config.smk @@ -0,0 +1,21 @@ + +args = sys.argv +config_path = ( + args[args.index("--configfile") + 1] + if "--configfile" in args + else default_configfile +) + + +dataset = config["dataset"] +ncores = config["ncores"] +ml_methods = config["ml_method"] if 'ml_method' in config else 'glmnet' +kfold = config["kfold"] if "kfold" in config else 5 +outcome_colname = config["outcome_colname"] if "outcome_colname" in config else None + +nseeds = config["nseeds"] if 'nseeds' in config else 1 +start_seed = 100 +seeds = range(start_seed, start_seed + nseeds) + +hyperparams = config["hyperparams"] if "hyperparams" in config else None +find_feature_importance = config["find_feature_importance"] if "find_feature_importance" in config else None diff --git a/workflow/rules/paramspace.smk b/workflow/rules/paramspace.smk new file mode 100644 index 0000000..901d2e9 --- /dev/null +++ b/workflow/rules/paramspace.smk @@ -0,0 +1,23 @@ + +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# Prepare Parameter Space +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +exclude_param_keys = config['exclude_param_keys'] +for k in exclude_param_keys: + config.pop(k, None) +config['seed'] = list(seeds) +conf_lists = {k:v for k,v in config.items() if type(v) == list} +params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) +for k in conf_lists.keys(): + config.pop(k) +for k, v in config.items(): + params_df[k] = v + +params_df = params_df[sorted(params_df.columns.tolist())] +paramspace = Paramspace(params_df, param_sep = "-") + +print("Wildcard pattern:", paramspace.wildcard_pattern) + +paramspace_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) +paramspace_no_seed = re.sub("/seed-{seed}/", "/", paramspace.wildcard_pattern) +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # From 58bdfc44cf4a536cad1b2bc33d3a3540e7ec425e Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 15:47:49 -0500 Subject: [PATCH 12/53] Get instance patterns without seed wildcard --- workflow/Snakefile | 7 ++++--- workflow/rules/combine.smk | 8 ++++---- workflow/rules/paramspace.smk | 9 +++++++-- workflow/rules/plot.smk | 8 ++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index c067d23..520f2cb 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -42,9 +42,10 @@ rule render_report: input: perf_plot=rules.plot_performance.output.plot, feat_plot=rules.plot_feature_importance.output.plot, - # hp_plot=expand( - # rules.plot_hp_performance.output.plot, ml_method=ml_methods - # ), + hp_plot=expand( + "figures/{params}/hp_performance.png", + params = instances_no_seed + ), bench_plot=rules.plot_benchmarks.output.plot, roc_plot=rules.plot_roc_curves.output.plot, rulegraph="figures/graphviz/rulegraph.png", diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 08f66e8..17e50fa 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -19,13 +19,13 @@ rule combine_results: rule combine_hp_performance: input: - rds=expand(f"results/{paramspace_tame_seed}/model.Rds", seed=seeds), + rds=expand(f"results/{wildcard_tame_seed}/model.Rds", seed=seeds), output: - rds=f"results/{paramspace_no_seed}/hp_performance_results.Rds", + rds=f"results/{wildcard_no_seed}/hp_performance_results.Rds", log: - f"log/{paramspace_no_seed}/combine_hp_perf.txt", + f"log/{wildcard_no_seed}/combine_hp_perf.txt", benchmark: - f"benchmarks/{paramspace_no_seed}/combine_hp_perf.txt" + f"benchmarks/{wildcard_no_seed}/combine_hp_perf.txt" resources: mem_mb=MEM_PER_GB * 16, conda: diff --git a/workflow/rules/paramspace.smk b/workflow/rules/paramspace.smk index 901d2e9..e6c1fb1 100644 --- a/workflow/rules/paramspace.smk +++ b/workflow/rules/paramspace.smk @@ -18,6 +18,11 @@ paramspace = Paramspace(params_df, param_sep = "-") print("Wildcard pattern:", paramspace.wildcard_pattern) -paramspace_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) -paramspace_no_seed = re.sub("/seed-{seed}/", "/", paramspace.wildcard_pattern) +wildcard_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) +wildcard_no_seed = re.sub("/seed-{seed}", "", paramspace.wildcard_pattern) +instances_no_seed = [re.sub("/seed-[a-zA-Z_0-9]*", "", i) for i in paramspace.instance_patterns] + +print('tame seed', wildcard_tame_seed) +print('no seed', wildcard_no_seed) +print('instances no seed', instances_no_seed) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 8e9deb4..436fd14 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -47,17 +47,17 @@ else: "../scripts/make_blank_plot.R" -rule plot_hp_performance: # TODO: modify this to take list of all hp results (without being precombined) and make a big facet_wrap or plot_grid +rule plot_hp_performance: input: - rds=f"results/{paramspace_no_seed}/hp_performance_results.Rds", + rds=f"results/{wildcard_no_seed}/hp_performance_results.Rds", output: plot=report( - "figures/hp_performance_{ml_method}.png", + f"figures/{wildcard_no_seed}/hp_performance.png", category="Performance", subcategory="Hyperparameter Tuning", ), log: - "log/plot_hp_perf_{ml_method}.txt", + f"log/{wildcard_no_seed}/plot_hp_perf.txt", conda: "../envs/mikropml.yml" script: From 8edca31c4e0359216dccea4398de1fe1322c7ad2 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 17:01:04 -0500 Subject: [PATCH 13/53] Create separate functions to tweak wildcards in paramspace --- workflow/Snakefile | 5 ++-- workflow/rules/combine.smk | 3 ++- workflow/rules/common.smk | 44 +++++++++++++++++++++++++++++++ workflow/rules/config.smk | 3 ++- workflow/rules/example-report.smk | 3 +++ workflow/rules/learn.smk | 34 +++++++++++++----------- workflow/rules/paramspace.smk | 18 +++---------- workflow/rules/plot.smk | 3 +++ 8 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 workflow/rules/common.smk diff --git a/workflow/Snakefile b/workflow/Snakefile index 520f2cb..94df964 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -2,7 +2,7 @@ import datetime import itertools as it import os import pandas as pd -from snakemake.utils import Paramspace, min_version +from snakemake.utils import min_version import sys @@ -21,6 +21,7 @@ configfile: default_configfile wildcard_constraints: kfold = '[0-9]+' +include: 'rules/common.smk' include: 'rules/config.smk' include: 'rules/paramspace.smk' @@ -44,7 +45,7 @@ rule render_report: feat_plot=rules.plot_feature_importance.output.plot, hp_plot=expand( "figures/{params}/hp_performance.png", - params = instances_no_seed + params = instances_drop_wildcard(paramspace, 'seed') ), bench_plot=rules.plot_benchmarks.output.plot, roc_plot=rules.plot_roc_curves.output.plot, diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 17e50fa..5816ab8 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -1,4 +1,5 @@ - +''' Combine results from individual `run_ml` jobs to prepare for plotting +''' rule combine_results: input: csv=expand( diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk new file mode 100644 index 0000000..bb58682 --- /dev/null +++ b/workflow/rules/common.smk @@ -0,0 +1,44 @@ +from collections.abc import Iterable +import re +from snakemake.utils import Paramspace + +test_wildcard_pattern = 'dataset-{dataset}/seed-{seed}/ml_method-{ml_method}' + +def pattern_tame_wildcard(paramspace, wildcard): + ''' Tame a wildcard in a pattern by doubling up curly braces on all other wildcards. This is useful for filling in values with expand(). + + :param paramspace: a paramspace + :type paramspace: class:`snakemake.utils.Paramspace` + :param wildcard: a wildcard in the paramspace + :type wildcard: str + + :return: wildcard pattern from paramspace without the wildcard + :rtype: str + ''' + return re.sub(f"{{((?!{wildcard})[a-zA-Z_0-9]*)}}", "{{\\1}}", paramspace.wildcard_pattern) + +def pattern_drop_wildcard(paramspace, wildcard): + """ Remove a wildcard from the wildcard pattern + + :param paramspace: a paramspace + :type paramspace: class:`snakemake.utils.Paramspace` + :param wildcard: a wildcard in the paramspace + :type wildcard: str + + :return: wildcard pattern from paramspace without the wildcard + :rtype: str + """ + return re.sub(f"{wildcard}{paramspace.param_sep}{{{wildcard}}}", "", paramspace.wildcard_pattern) + +def instances_drop_wildcard(paramspace, wildcard): + ''' Remove a wildcard from instance patterns + + :param paramspace: a paramspace + :type paramspace: class:`snakemake.utils.Paramspace` + :param wildcard: a wildcard in the paramspace + :type wildcard: str + + :return: list of instance patterns from paramspace without the wildcard + :rtype: list + ''' + return [re.sub(f"{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i) for i in paramspace.instance_patterns] diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index d334b54..dc1acfe 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -1,4 +1,5 @@ - +''' Extract variables from the configfile and set default values +''' args = sys.argv config_path = ( args[args.index("--configfile") + 1] diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index a5ead0a..60e5862 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -1,3 +1,6 @@ +''' Create an example report +''' + rule copy_example_figures: input: figs=[ diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index f04295d..75e8e35 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -1,3 +1,6 @@ +''' Preprocess data, train ML models, calculate performance, and find feature importance +''' + rule preprocess_data: input: csv=f"data/{dataset}.csv", @@ -40,38 +43,39 @@ rule run_ml: script: "../scripts/train_ml.R" -rule find_feature_importance: + +rule calc_model_sensspec: input: model=rules.run_ml.output.model, test=rules.run_ml.output.test, output: - feat=f"results/{paramspace.wildcard_pattern}/feature_importance.csv", - log: - f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", + csv=f"results/{paramspace.wildcard_pattern}/sensspec.csv", params: outcome_colname=outcome_colname, - method="{ml_method}", - seed="{seed}", - threads: ncores - resources: - mem_mb=MEM_PER_GB * 1, + log: + f"log/{paramspace.wildcard_pattern}/calc_model_sensspec.txt", conda: "../envs/mikropml.yml" script: - "../scripts/find_feature_importance.R" + "../scripts/calc_model_sensspec.R" -rule calc_model_sensspec: +rule find_feature_importance: input: model=rules.run_ml.output.model, test=rules.run_ml.output.test, output: - csv=f"results/{paramspace.wildcard_pattern}/sensspec.csv", + feat=f"results/{paramspace.wildcard_pattern}/feature_importance.csv", + log: + f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", params: outcome_colname=outcome_colname, - log: - f"log/{paramspace.wildcard_pattern}/calc_model_sensspec.txt", + method="{ml_method}", + seed="{seed}", + threads: ncores + resources: + mem_mb=MEM_PER_GB * 1, conda: "../envs/mikropml.yml" script: - "../scripts/calc_model_sensspec.R" + "../scripts/find_feature_importance.R" diff --git a/workflow/rules/paramspace.smk b/workflow/rules/paramspace.smk index e6c1fb1..80e6607 100644 --- a/workflow/rules/paramspace.smk +++ b/workflow/rules/paramspace.smk @@ -1,7 +1,5 @@ - -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# Prepare Parameter Space -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +''' Prepare the parameter space based on the config dictionary +''' exclude_param_keys = config['exclude_param_keys'] for k in exclude_param_keys: config.pop(k, None) @@ -16,13 +14,5 @@ for k, v in config.items(): params_df = params_df[sorted(params_df.columns.tolist())] paramspace = Paramspace(params_df, param_sep = "-") -print("Wildcard pattern:", paramspace.wildcard_pattern) - -wildcard_tame_seed = re.sub("{((?!seed)[a-zA-Z_0-9]*)}", "{{\\1}}", paramspace.wildcard_pattern) -wildcard_no_seed = re.sub("/seed-{seed}", "", paramspace.wildcard_pattern) -instances_no_seed = [re.sub("/seed-[a-zA-Z_0-9]*", "", i) for i in paramspace.instance_patterns] - -print('tame seed', wildcard_tame_seed) -print('no seed', wildcard_no_seed) -print('instances no seed', instances_no_seed) -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') +wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') \ No newline at end of file diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 436fd14..40135e6 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -1,3 +1,6 @@ +''' Plot ML results +''' + rule plot_performance: input: csv="results/performance-results.csv", From 4c245b58e38610f557c3ee800eef196e9f950b39 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 18:11:36 -0500 Subject: [PATCH 14/53] Move paramspace-related functions to separate script and write unit tests --- workflow/Snakefile | 4 ++-- workflow/__init__.py | 0 workflow/envs/github-actions.yml | 2 ++ workflow/rules/config.smk | 1 - workflow/rules/paramspace.smk | 18 +++++++++++---- workflow/scripts/__init__.py | 0 .../common.smk => scripts/functions.py} | 8 ++----- workflow/scripts/test_functions.py | 23 +++++++++++++++++++ 8 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 workflow/__init__.py create mode 100644 workflow/scripts/__init__.py rename workflow/{rules/common.smk => scripts/functions.py} (77%) create mode 100644 workflow/scripts/test_functions.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 94df964..608bcf4 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -2,8 +2,9 @@ import datetime import itertools as it import os import pandas as pd -from snakemake.utils import min_version +from snakemake.utils import min_version, Paramspace import sys +from scripts.functions import * min_version("6.12.3") @@ -21,7 +22,6 @@ configfile: default_configfile wildcard_constraints: kfold = '[0-9]+' -include: 'rules/common.smk' include: 'rules/config.smk' include: 'rules/paramspace.smk' diff --git a/workflow/__init__.py b/workflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/workflow/envs/github-actions.yml b/workflow/envs/github-actions.yml index 0ad26a9..266b249 100644 --- a/workflow/envs/github-actions.yml +++ b/workflow/envs/github-actions.yml @@ -6,6 +6,8 @@ channels: - r dependencies: - black + - pytest + - pytest-parallel - python=3.11 - r-base=4 - r-styler diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index dc1acfe..6cab613 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -7,7 +7,6 @@ config_path = ( else default_configfile ) - dataset = config["dataset"] ncores = config["ncores"] ml_methods = config["ml_method"] if 'ml_method' in config else 'glmnet' diff --git a/workflow/rules/paramspace.smk b/workflow/rules/paramspace.smk index 80e6607..c48d5b8 100644 --- a/workflow/rules/paramspace.smk +++ b/workflow/rules/paramspace.smk @@ -1,18 +1,28 @@ ''' Prepare the parameter space based on the config dictionary ''' +# exclude certain config items from the Paramspace exclude_param_keys = config['exclude_param_keys'] for k in exclude_param_keys: config.pop(k, None) + +# add list of seeds to config config['seed'] = list(seeds) + +# get lists from config conf_lists = {k:v for k,v in config.items() if type(v) == list} +# get a dataframe with all vs all values in the list. +# this is analogous to `param.grid()` in R. params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) -for k in conf_lists.keys(): - config.pop(k) +# add all non-list config items to the dataframe of dot products for k, v in config.items(): - params_df[k] = v - + if type(v) != list: + params_df[k] = v +# sort columns ascii-betically params_df = params_df[sorted(params_df.columns.tolist())] +# build the paramspace with the dataframe paramspace = Paramspace(params_df, param_sep = "-") +# wildcard pattern without seed. needed for rule `combine_hp_performance` wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') +# wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') \ No newline at end of file diff --git a/workflow/scripts/__init__.py b/workflow/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/workflow/rules/common.smk b/workflow/scripts/functions.py similarity index 77% rename from workflow/rules/common.smk rename to workflow/scripts/functions.py index bb58682..b497b8c 100644 --- a/workflow/rules/common.smk +++ b/workflow/scripts/functions.py @@ -1,8 +1,4 @@ -from collections.abc import Iterable import re -from snakemake.utils import Paramspace - -test_wildcard_pattern = 'dataset-{dataset}/seed-{seed}/ml_method-{ml_method}' def pattern_tame_wildcard(paramspace, wildcard): ''' Tame a wildcard in a pattern by doubling up curly braces on all other wildcards. This is useful for filling in values with expand(). @@ -28,7 +24,7 @@ def pattern_drop_wildcard(paramspace, wildcard): :return: wildcard pattern from paramspace without the wildcard :rtype: str """ - return re.sub(f"{wildcard}{paramspace.param_sep}{{{wildcard}}}", "", paramspace.wildcard_pattern) + return re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}{{{wildcard}}}", "", paramspace.wildcard_pattern).strip('/') def instances_drop_wildcard(paramspace, wildcard): ''' Remove a wildcard from instance patterns @@ -41,4 +37,4 @@ def instances_drop_wildcard(paramspace, wildcard): :return: list of instance patterns from paramspace without the wildcard :rtype: list ''' - return [re.sub(f"{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i) for i in paramspace.instance_patterns] + return [re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip("/") for i in paramspace.instance_patterns] diff --git a/workflow/scripts/test_functions.py b/workflow/scripts/test_functions.py new file mode 100644 index 0000000..01c2c7f --- /dev/null +++ b/workflow/scripts/test_functions.py @@ -0,0 +1,23 @@ +import pandas as pd +from snakemake.utils import Paramspace + +from .functions import * + +config = {'dataset': 'otu_large', + 'seed': list(range(2)), + 'ml_method': ['glmnet', 'rf']} +params = Paramspace(pd.DataFrame(config), param_sep = '-') + +def test_pattern_tame_wildcard(): + assert pattern_tame_wildcard(params, 'seed') == 'dataset-{{dataset}}/seed-{seed}/ml_method-{{ml_method}}' + assert pattern_tame_wildcard(params, 'dataset') == 'dataset-{dataset}/seed-{{seed}}/ml_method-{{ml_method}}' + +def test_pattern_drop_wildcard(): + assert pattern_drop_wildcard(params, 'seed') == 'dataset-{dataset}/ml_method-{ml_method}' + assert pattern_drop_wildcard(params, 'dataset') == 'seed-{seed}/ml_method-{ml_method}' + assert pattern_drop_wildcard(params, 'ml_method') == 'dataset-{dataset}/seed-{seed}' + +def test_instances_drop_wildcard(): + assert instances_drop_wildcard(params, 'seed') == ['dataset-otu_large/ml_method-glmnet', 'dataset-otu_large/ml_method-rf'] + assert instances_drop_wildcard(params, 'dataset') == ['seed-0/ml_method-glmnet', 'seed-1/ml_method-rf'] + assert instances_drop_wildcard(params, 'ml_method') == ['dataset-otu_large/seed-0', 'dataset-otu_large/seed-1'] From 57b8f82269f53f6b01f9ab90d01b82c8a0df9e34 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 18:12:00 -0500 Subject: [PATCH 15/53] Use mambaforge for deps & test files in workflow/scripts/ --- .github/workflows/tests.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d612ec2..bda850c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,13 +21,13 @@ jobs: with: persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@v4 + - uses: conda-incubator/setup-miniconda@v2 with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest pytest-parallel + python-version: 3.11 + miniforge-variant: Mambaforge + miniforge-version: latest + activate-environment: github-actions + environment-file: workflow/envs/github-actions.yml - name: Lint workflow uses: snakemake/snakemake-github-action@v1.24.0 with: @@ -42,4 +42,7 @@ jobs: args: "--cores 2 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache --show-failed-logs --all-temp --configfile config/test.yaml" # - name: Test with pytest # run: | -# pytest --workers 2 .tests/ \ No newline at end of file +# pytest --workers 2 .tests/ + - name: Test with pytest + run: | + pytest --workers 2 workflow/scripts/ \ No newline at end of file From 50800e275c69ca707fdbc5f13d7707c11da3dd00 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 18:42:33 -0500 Subject: [PATCH 16/53] Write get_paramspace_from_config() Also fix instances_drop_wildcard() so it returns a unique set --- workflow/Snakefile | 7 +--- workflow/rules/config.smk | 8 +++++ workflow/rules/paramspace.smk | 28 ---------------- workflow/scripts/functions.py | 53 ++++++++++++++++++++++++++++-- workflow/scripts/test_functions.py | 25 ++++++++++++-- 5 files changed, 81 insertions(+), 40 deletions(-) delete mode 100644 workflow/rules/paramspace.smk diff --git a/workflow/Snakefile b/workflow/Snakefile index 608bcf4..9fecd66 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,8 +1,6 @@ import datetime -import itertools as it import os -import pandas as pd -from snakemake.utils import min_version, Paramspace +from snakemake.utils import min_version import sys from scripts.functions import * @@ -23,14 +21,11 @@ wildcard_constraints: kfold = '[0-9]+' include: 'rules/config.smk' -include: 'rules/paramspace.smk' - include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" #include: "rules/example-report.smk" - report: "report/workflow.rst" diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 6cab613..898ac84 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -16,6 +16,14 @@ outcome_colname = config["outcome_colname"] if "outcome_colname" in config else nseeds = config["nseeds"] if 'nseeds' in config else 1 start_seed = 100 seeds = range(start_seed, start_seed + nseeds) +config['seed'] = list(seeds) hyperparams = config["hyperparams"] if "hyperparams" in config else None find_feature_importance = config["find_feature_importance"] if "find_feature_importance" in config else None + +# parameter space based on configfile +paramspace = get_paramspace_from_config(config) +# wildcard pattern without seed. needed for rule `combine_hp_performance` +wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') +# wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` +wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') \ No newline at end of file diff --git a/workflow/rules/paramspace.smk b/workflow/rules/paramspace.smk deleted file mode 100644 index c48d5b8..0000000 --- a/workflow/rules/paramspace.smk +++ /dev/null @@ -1,28 +0,0 @@ -''' Prepare the parameter space based on the config dictionary -''' -# exclude certain config items from the Paramspace -exclude_param_keys = config['exclude_param_keys'] -for k in exclude_param_keys: - config.pop(k, None) - -# add list of seeds to config -config['seed'] = list(seeds) - -# get lists from config -conf_lists = {k:v for k,v in config.items() if type(v) == list} -# get a dataframe with all vs all values in the list. -# this is analogous to `param.grid()` in R. -params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) -# add all non-list config items to the dataframe of dot products -for k, v in config.items(): - if type(v) != list: - params_df[k] = v -# sort columns ascii-betically -params_df = params_df[sorted(params_df.columns.tolist())] -# build the paramspace with the dataframe -paramspace = Paramspace(params_df, param_sep = "-") - -# wildcard pattern without seed. needed for rule `combine_hp_performance` -wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') -# wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` -wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') \ No newline at end of file diff --git a/workflow/scripts/functions.py b/workflow/scripts/functions.py index b497b8c..5910cb6 100644 --- a/workflow/scripts/functions.py +++ b/workflow/scripts/functions.py @@ -1,4 +1,8 @@ +from collections.abc import Iterable +import itertools as it +import pandas as pd import re +from snakemake.utils import Paramspace def pattern_tame_wildcard(paramspace, wildcard): ''' Tame a wildcard in a pattern by doubling up curly braces on all other wildcards. This is useful for filling in values with expand(). @@ -34,7 +38,50 @@ def instances_drop_wildcard(paramspace, wildcard): :param wildcard: a wildcard in the paramspace :type wildcard: str - :return: list of instance patterns from paramspace without the wildcard - :rtype: list + :return: set of instance patterns from paramspace without the wildcard + :rtype: set ''' - return [re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip("/") for i in paramspace.instance_patterns] + return {re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip("/") for i in paramspace.instance_patterns} + +def get_paramspace_from_config(config): + ''' Prepare the parameter space based on the config dictionary + + :param config: configuration dictionary + :type config: dict + + :return: parameter space + :rtype: class:`snakemake.utils.Paramspace` + ''' + # exclude certain config items from the Paramspace + exclude_param_keys = config['exclude_param_keys'] if 'exclude_param_keys' in config and isinstance(config['exclude_param_keys'], Iterable) else list() + for k in exclude_param_keys: + config.pop(k, None) + + # get lists from config + conf_lists = {k:v for k,v in config.items() if type(v) == list} + # get a data frame with all-vs-all values in the lists. + # this is analogous to `param.grid()` in R. + params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) + # add all non-list config items to the data frame. + for k, v in config.items(): + if type(v) != list: + params_df[k] = v + # sort columns ascii-betically + params_df = params_df[sorted(params_df.columns.tolist())] + # build the paramspace with the data frame + paramspace = Paramspace(params_df, param_sep = "-") + return paramspace + +def set_default(config, key, default): + """ Get a value from config if the key exists, + otherwise return the default value. + + :param config: configuration dictionary + :type config: dict + :param key: a key that may or may not be in the dictionary + :type key: any hashable type + :param default: the default value to return if the key is not in config + :type default: any type + :return: the value of the key if it's in config, otherwise the dfeault value + """ + return config[key] if key in config else default \ No newline at end of file diff --git a/workflow/scripts/test_functions.py b/workflow/scripts/test_functions.py index 01c2c7f..8f9fef8 100644 --- a/workflow/scripts/test_functions.py +++ b/workflow/scripts/test_functions.py @@ -18,6 +18,25 @@ def test_pattern_drop_wildcard(): assert pattern_drop_wildcard(params, 'ml_method') == 'dataset-{dataset}/seed-{seed}' def test_instances_drop_wildcard(): - assert instances_drop_wildcard(params, 'seed') == ['dataset-otu_large/ml_method-glmnet', 'dataset-otu_large/ml_method-rf'] - assert instances_drop_wildcard(params, 'dataset') == ['seed-0/ml_method-glmnet', 'seed-1/ml_method-rf'] - assert instances_drop_wildcard(params, 'ml_method') == ['dataset-otu_large/seed-0', 'dataset-otu_large/seed-1'] + assert instances_drop_wildcard(params, 'seed') == {'dataset-otu_large/ml_method-glmnet', 'dataset-otu_large/ml_method-rf'} + assert instances_drop_wildcard(params, 'dataset') == {'seed-0/ml_method-glmnet', 'seed-1/ml_method-rf'} + assert instances_drop_wildcard(params, 'ml_method') == {'dataset-otu_large/seed-0', 'dataset-otu_large/seed-1'} + + conf2 = {'dataset': 'otu_large', + 'seed': list(range(2)), + 'kfold': 5, + 'ml_method': ['glmnet', 'rf']} + p2 = get_paramspace_from_config(conf2) + assert instances_drop_wildcard(p2, 'seed') == {'dataset-otu_large/kfold-5/ml_method-glmnet', 'dataset-otu_large/kfold-5/ml_method-rf'} + +def test_get_paramspace_from_config(): + config = {'dataset': 'otu_large', + 'exclude_param_keys': ['exclude_param_keys'], + 'seed': list(range(4)), + 'ml_method': ['glmnet', 'rf']} + p = get_paramspace_from_config(config) + assert p.wildcard_pattern == 'dataset-{dataset}/ml_method-{ml_method}/seed-{seed}' + assert list(p.instance_patterns) == ['dataset-otu_large/ml_method-glmnet/seed-0', 'dataset-otu_large/ml_method-rf/seed-0', 'dataset-otu_large/ml_method-glmnet/seed-1', 'dataset-otu_large/ml_method-rf/seed-1', 'dataset-otu_large/ml_method-glmnet/seed-2', 'dataset-otu_large/ml_method-rf/seed-2', 'dataset-otu_large/ml_method-glmnet/seed-3', 'dataset-otu_large/ml_method-rf/seed-3'] + +def test_set_default(): + pass # TODO \ No newline at end of file From afbb45bacf280980924bbf7b19ae1d31d5c05bf8 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 18:47:25 -0500 Subject: [PATCH 17/53] Use set_default() for more DRY code --- workflow/rules/config.smk | 15 +++++++-------- workflow/scripts/test_functions.py | 3 ++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 898ac84..80b3905 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -9,21 +9,20 @@ config_path = ( dataset = config["dataset"] ncores = config["ncores"] -ml_methods = config["ml_method"] if 'ml_method' in config else 'glmnet' -kfold = config["kfold"] if "kfold" in config else 5 -outcome_colname = config["outcome_colname"] if "outcome_colname" in config else None +ml_methods = set_default(config, 'ml_method', 'glmnet') +kfold = set_default(config, 'kfold', 5) +outcome_colname = set_default(config, 'outcome_colname', None) +hyperparams = set_default(config, 'hyperparams', None) +find_feature_importance = set_default(config, 'hyperparams', False) -nseeds = config["nseeds"] if 'nseeds' in config else 1 +nseeds = set_default(config, 'nseeds', 1) start_seed = 100 seeds = range(start_seed, start_seed + nseeds) config['seed'] = list(seeds) -hyperparams = config["hyperparams"] if "hyperparams" in config else None -find_feature_importance = config["find_feature_importance"] if "find_feature_importance" in config else None - # parameter space based on configfile paramspace = get_paramspace_from_config(config) # wildcard pattern without seed. needed for rule `combine_hp_performance` wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') # wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` -wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') \ No newline at end of file +wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') diff --git a/workflow/scripts/test_functions.py b/workflow/scripts/test_functions.py index 8f9fef8..c242673 100644 --- a/workflow/scripts/test_functions.py +++ b/workflow/scripts/test_functions.py @@ -39,4 +39,5 @@ def test_get_paramspace_from_config(): assert list(p.instance_patterns) == ['dataset-otu_large/ml_method-glmnet/seed-0', 'dataset-otu_large/ml_method-rf/seed-0', 'dataset-otu_large/ml_method-glmnet/seed-1', 'dataset-otu_large/ml_method-rf/seed-1', 'dataset-otu_large/ml_method-glmnet/seed-2', 'dataset-otu_large/ml_method-rf/seed-2', 'dataset-otu_large/ml_method-glmnet/seed-3', 'dataset-otu_large/ml_method-rf/seed-3'] def test_set_default(): - pass # TODO \ No newline at end of file + assert set_default(config, 'dataset', None) == 'otu_large' + assert set_default(config, 'not_in_dict', 'value') == 'value' \ No newline at end of file From e6b5e2a10935d77d296b215d62fa529668af37c2 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 19:04:30 -0500 Subject: [PATCH 18/53] Fix indent --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b4771f2..0fa4b43 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,6 @@ jobs: # - name: Test with pytest # run: | # pytest --workers 2 .tests/ - - name: Test with pytest - run: | - pytest --workers 2 workflow/scripts/ \ No newline at end of file + - name: Test with pytest + run: | + pytest --workers 2 workflow/scripts/ \ No newline at end of file From 2aec70fac5acb427546539984947aeedc11eb641 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 19:41:47 -0500 Subject: [PATCH 19/53] Fix typo for find_feature_importance Have to use output filepath, not rules.output, because the plot is blank when find_feat_imp is False --- workflow/Snakefile | 2 +- workflow/rules/config.smk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 28d974c..13e3507 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -37,7 +37,7 @@ rule targets: rule render_report: input: perf_plot=rules.plot_performance.output.plot, - feat_plot=rules.plot_feature_importance.output.plot, + feat_plot="figures/feature_importance.png", hp_plot=expand( "figures/{params}/hp_performance.png", params = instances_drop_wildcard(paramspace, 'seed') diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 80b3905..3645083 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -13,7 +13,7 @@ ml_methods = set_default(config, 'ml_method', 'glmnet') kfold = set_default(config, 'kfold', 5) outcome_colname = set_default(config, 'outcome_colname', None) hyperparams = set_default(config, 'hyperparams', None) -find_feature_importance = set_default(config, 'hyperparams', False) +find_feature_importance = set_default(config, 'find_feature_importance', False) nseeds = set_default(config, 'nseeds', 1) start_seed = 100 From 7bb825509d44f8bd641ace5509ceecd7fd8d1e7f Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 19:43:32 -0500 Subject: [PATCH 20/53] Use 'include' instead of 'import' for snakedeploy Otherwise, will get a ModuleNotFoundError when deploying this module with snakedeploy --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 13e3507..b4a236a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -2,7 +2,6 @@ import datetime import os from snakemake.utils import min_version import sys -from scripts.functions import * min_version("6.12.3") @@ -20,6 +19,7 @@ configfile: default_configfile wildcard_constraints: kfold = '[0-9]+' +include: 'scripts/functions.py' include: 'rules/config.smk' include: "rules/learn.smk" include: "rules/combine.smk" From f6cd873ecc7849911967adc3f68d088d2830ab87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Jan 2023 00:46:09 +0000 Subject: [PATCH 21/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/Snakefile | 22 ++++--- workflow/rules/combine.smk | 11 ++-- workflow/rules/config.smk | 22 +++---- workflow/rules/example-report.smk | 13 +++-- workflow/rules/learn.smk | 5 +- workflow/rules/plot.smk | 5 +- workflow/scripts/functions.py | 63 +++++++++++++------- workflow/scripts/test_functions.py | 93 +++++++++++++++++++++--------- 8 files changed, 152 insertions(+), 82 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index b4a236a..4bef17e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -14,17 +14,23 @@ default_configfile = "config/config.yaml" MEM_PER_GB = 1024 + configfile: default_configfile + wildcard_constraints: - kfold = '[0-9]+' + kfold="[0-9]+", + -include: 'scripts/functions.py' -include: 'rules/config.smk' +include: "scripts/functions.py" +include: "rules/config.smk" include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" -#include: "rules/example-report.smk" + + +# include: "rules/example-report.smk" + report: "report/workflow.rst" @@ -39,8 +45,8 @@ rule render_report: perf_plot=rules.plot_performance.output.plot, feat_plot="figures/feature_importance.png", hp_plot=expand( - "figures/{params}/hp_performance.png", - params = instances_drop_wildcard(paramspace, 'seed') + "figures/{params}/hp_performance.png", + params=instances_drop_wildcard(paramspace, "seed"), ), bench_plot=rules.plot_benchmarks.output.plot, roc_plot=rules.plot_roc_curves.output.plot, @@ -65,8 +71,8 @@ rule render_report: rule archive: input: - expand(rules.render_report.input, dataset = dataset), - expand(rules.render_report.output, dataset = dataset), + expand(rules.render_report.input, dataset=dataset), + expand(rules.render_report.output, dataset=dataset), expand( "results/{rtype}-results.csv", rtype=["performance", "feature_importance", "benchmarks", "sensspec"], diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 5816ab8..e734f5c 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -1,11 +1,10 @@ -''' Combine results from individual `run_ml` jobs to prepare for plotting -''' +""" Combine results from individual `run_ml` jobs to prepare for plotting +""" + + rule combine_results: input: - csv=expand( - "results/{params}/{{type}}.csv", - params=paramspace.instance_patterns - ), + csv=expand("results/{params}/{{type}}.csv", params=paramspace.instance_patterns), output: csv="results/{type}-results.csv", log: diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 3645083..ef7f1d3 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -1,5 +1,5 @@ -''' Extract variables from the configfile and set default values -''' +""" Extract variables from the configfile and set default values +""" args = sys.argv config_path = ( args[args.index("--configfile") + 1] @@ -9,20 +9,20 @@ config_path = ( dataset = config["dataset"] ncores = config["ncores"] -ml_methods = set_default(config, 'ml_method', 'glmnet') -kfold = set_default(config, 'kfold', 5) -outcome_colname = set_default(config, 'outcome_colname', None) -hyperparams = set_default(config, 'hyperparams', None) -find_feature_importance = set_default(config, 'find_feature_importance', False) +ml_methods = set_default(config, "ml_method", "glmnet") +kfold = set_default(config, "kfold", 5) +outcome_colname = set_default(config, "outcome_colname", None) +hyperparams = set_default(config, "hyperparams", None) +find_feature_importance = set_default(config, "find_feature_importance", False) -nseeds = set_default(config, 'nseeds', 1) +nseeds = set_default(config, "nseeds", 1) start_seed = 100 seeds = range(start_seed, start_seed + nseeds) -config['seed'] = list(seeds) +config["seed"] = list(seeds) # parameter space based on configfile paramspace = get_paramspace_from_config(config) # wildcard pattern without seed. needed for rule `combine_hp_performance` -wildcard_no_seed = pattern_drop_wildcard(paramspace, 'seed') +wildcard_no_seed = pattern_drop_wildcard(paramspace, "seed") # wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` -wildcard_tame_seed = pattern_tame_wildcard(paramspace, 'seed') +wildcard_tame_seed = pattern_tame_wildcard(paramspace, "seed") diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index 60e5862..dca8d59 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -1,13 +1,13 @@ -''' Create an example report -''' +""" Create an example report +""" + rule copy_example_figures: input: figs=[ rules.plot_performance.output.plot, rules.plot_feature_importance.output.plot, - expand(rules.plot_hp_performance.output.plot, - ml_method=ml_methods), + expand(rules.plot_hp_performance.output.plot, ml_method=ml_methods), rules.plot_benchmarks.output.plot, rules.plot_roc_curves.output.plot, "figures/graphviz/rulegraph.png", @@ -16,8 +16,9 @@ rule copy_example_figures: perf_plot="figures/example/performance.png", feat_plot="figures/example/feature_importance.png", bench_plot="figures/example/benchmarks.png", - hp_plot=expand("figures/example/hp_performance_{ml_method}.png", - ml_method=ml_methods), + hp_plot=expand( + "figures/example/hp_performance_{ml_method}.png", ml_method=ml_methods + ), roc_plot="figures/example/roc_curves.png", rulegraph="figures/example/rulegraph.png", log: diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 75e8e35..1e97ed2 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -1,5 +1,6 @@ -''' Preprocess data, train ML models, calculate performance, and find feature importance -''' +""" Preprocess data, train ML models, calculate performance, and find feature importance +""" + rule preprocess_data: input: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 40135e6..0318a06 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -1,5 +1,6 @@ -''' Plot ML results -''' +""" Plot ML results +""" + rule plot_performance: input: diff --git a/workflow/scripts/functions.py b/workflow/scripts/functions.py index 5910cb6..8a7326d 100644 --- a/workflow/scripts/functions.py +++ b/workflow/scripts/functions.py @@ -4,64 +4,86 @@ import re from snakemake.utils import Paramspace + def pattern_tame_wildcard(paramspace, wildcard): - ''' Tame a wildcard in a pattern by doubling up curly braces on all other wildcards. This is useful for filling in values with expand(). + """Tame a wildcard in a pattern by doubling up curly braces on all other wildcards. This is useful for filling in values with expand(). :param paramspace: a paramspace - :type paramspace: class:`snakemake.utils.Paramspace` + :type paramspace: class:`snakemake.utils.Paramspace` :param wildcard: a wildcard in the paramspace :type wildcard: str :return: wildcard pattern from paramspace without the wildcard :rtype: str - ''' - return re.sub(f"{{((?!{wildcard})[a-zA-Z_0-9]*)}}", "{{\\1}}", paramspace.wildcard_pattern) + """ + return re.sub( + f"{{((?!{wildcard})[a-zA-Z_0-9]*)}}", "{{\\1}}", paramspace.wildcard_pattern + ) + def pattern_drop_wildcard(paramspace, wildcard): - """ Remove a wildcard from the wildcard pattern + """Remove a wildcard from the wildcard pattern :param paramspace: a paramspace - :type paramspace: class:`snakemake.utils.Paramspace` + :type paramspace: class:`snakemake.utils.Paramspace` :param wildcard: a wildcard in the paramspace :type wildcard: str :return: wildcard pattern from paramspace without the wildcard :rtype: str """ - return re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}{{{wildcard}}}", "", paramspace.wildcard_pattern).strip('/') + return re.sub( + f"/{{0,1}}{wildcard}{paramspace.param_sep}{{{wildcard}}}", + "", + paramspace.wildcard_pattern, + ).strip("/") + def instances_drop_wildcard(paramspace, wildcard): - ''' Remove a wildcard from instance patterns - + """Remove a wildcard from instance patterns + :param paramspace: a paramspace - :type paramspace: class:`snakemake.utils.Paramspace` + :type paramspace: class:`snakemake.utils.Paramspace` :param wildcard: a wildcard in the paramspace :type wildcard: str :return: set of instance patterns from paramspace without the wildcard :rtype: set - ''' - return {re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip("/") for i in paramspace.instance_patterns} + """ + return { + re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip( + "/" + ) + for i in paramspace.instance_patterns + } + def get_paramspace_from_config(config): - ''' Prepare the parameter space based on the config dictionary + """Prepare the parameter space based on the config dictionary :param config: configuration dictionary :type config: dict :return: parameter space :rtype: class:`snakemake.utils.Paramspace` - ''' + """ # exclude certain config items from the Paramspace - exclude_param_keys = config['exclude_param_keys'] if 'exclude_param_keys' in config and isinstance(config['exclude_param_keys'], Iterable) else list() + exclude_param_keys = ( + config["exclude_param_keys"] + if "exclude_param_keys" in config + and isinstance(config["exclude_param_keys"], Iterable) + else list() + ) for k in exclude_param_keys: config.pop(k, None) # get lists from config - conf_lists = {k:v for k,v in config.items() if type(v) == list} + conf_lists = {k: v for k, v in config.items() if type(v) == list} # get a data frame with all-vs-all values in the lists. # this is analogous to `param.grid()` in R. - params_df = pd.DataFrame(list(it.product(*[v for v in conf_lists.values()])), columns = conf_lists.keys()) + params_df = pd.DataFrame( + list(it.product(*[v for v in conf_lists.values()])), columns=conf_lists.keys() + ) # add all non-list config items to the data frame. for k, v in config.items(): if type(v) != list: @@ -69,11 +91,12 @@ def get_paramspace_from_config(config): # sort columns ascii-betically params_df = params_df[sorted(params_df.columns.tolist())] # build the paramspace with the data frame - paramspace = Paramspace(params_df, param_sep = "-") + paramspace = Paramspace(params_df, param_sep="-") return paramspace + def set_default(config, key, default): - """ Get a value from config if the key exists, + """Get a value from config if the key exists, otherwise return the default value. :param config: configuration dictionary @@ -84,4 +107,4 @@ def set_default(config, key, default): :type default: any type :return: the value of the key if it's in config, otherwise the dfeault value """ - return config[key] if key in config else default \ No newline at end of file + return config[key] if key in config else default diff --git a/workflow/scripts/test_functions.py b/workflow/scripts/test_functions.py index c242673..bb2cdd3 100644 --- a/workflow/scripts/test_functions.py +++ b/workflow/scripts/test_functions.py @@ -3,41 +3,80 @@ from .functions import * -config = {'dataset': 'otu_large', - 'seed': list(range(2)), - 'ml_method': ['glmnet', 'rf']} -params = Paramspace(pd.DataFrame(config), param_sep = '-') +config = {"dataset": "otu_large", "seed": list(range(2)), "ml_method": ["glmnet", "rf"]} +params = Paramspace(pd.DataFrame(config), param_sep="-") + def test_pattern_tame_wildcard(): - assert pattern_tame_wildcard(params, 'seed') == 'dataset-{{dataset}}/seed-{seed}/ml_method-{{ml_method}}' - assert pattern_tame_wildcard(params, 'dataset') == 'dataset-{dataset}/seed-{{seed}}/ml_method-{{ml_method}}' + assert ( + pattern_tame_wildcard(params, "seed") + == "dataset-{{dataset}}/seed-{seed}/ml_method-{{ml_method}}" + ) + assert ( + pattern_tame_wildcard(params, "dataset") + == "dataset-{dataset}/seed-{{seed}}/ml_method-{{ml_method}}" + ) + def test_pattern_drop_wildcard(): - assert pattern_drop_wildcard(params, 'seed') == 'dataset-{dataset}/ml_method-{ml_method}' - assert pattern_drop_wildcard(params, 'dataset') == 'seed-{seed}/ml_method-{ml_method}' - assert pattern_drop_wildcard(params, 'ml_method') == 'dataset-{dataset}/seed-{seed}' + assert ( + pattern_drop_wildcard(params, "seed") + == "dataset-{dataset}/ml_method-{ml_method}" + ) + assert ( + pattern_drop_wildcard(params, "dataset") == "seed-{seed}/ml_method-{ml_method}" + ) + assert pattern_drop_wildcard(params, "ml_method") == "dataset-{dataset}/seed-{seed}" + def test_instances_drop_wildcard(): - assert instances_drop_wildcard(params, 'seed') == {'dataset-otu_large/ml_method-glmnet', 'dataset-otu_large/ml_method-rf'} - assert instances_drop_wildcard(params, 'dataset') == {'seed-0/ml_method-glmnet', 'seed-1/ml_method-rf'} - assert instances_drop_wildcard(params, 'ml_method') == {'dataset-otu_large/seed-0', 'dataset-otu_large/seed-1'} - - conf2 = {'dataset': 'otu_large', - 'seed': list(range(2)), - 'kfold': 5, - 'ml_method': ['glmnet', 'rf']} + assert instances_drop_wildcard(params, "seed") == { + "dataset-otu_large/ml_method-glmnet", + "dataset-otu_large/ml_method-rf", + } + assert instances_drop_wildcard(params, "dataset") == { + "seed-0/ml_method-glmnet", + "seed-1/ml_method-rf", + } + assert instances_drop_wildcard(params, "ml_method") == { + "dataset-otu_large/seed-0", + "dataset-otu_large/seed-1", + } + + conf2 = { + "dataset": "otu_large", + "seed": list(range(2)), + "kfold": 5, + "ml_method": ["glmnet", "rf"], + } p2 = get_paramspace_from_config(conf2) - assert instances_drop_wildcard(p2, 'seed') == {'dataset-otu_large/kfold-5/ml_method-glmnet', 'dataset-otu_large/kfold-5/ml_method-rf'} + assert instances_drop_wildcard(p2, "seed") == { + "dataset-otu_large/kfold-5/ml_method-glmnet", + "dataset-otu_large/kfold-5/ml_method-rf", + } + def test_get_paramspace_from_config(): - config = {'dataset': 'otu_large', - 'exclude_param_keys': ['exclude_param_keys'], - 'seed': list(range(4)), - 'ml_method': ['glmnet', 'rf']} + config = { + "dataset": "otu_large", + "exclude_param_keys": ["exclude_param_keys"], + "seed": list(range(4)), + "ml_method": ["glmnet", "rf"], + } p = get_paramspace_from_config(config) - assert p.wildcard_pattern == 'dataset-{dataset}/ml_method-{ml_method}/seed-{seed}' - assert list(p.instance_patterns) == ['dataset-otu_large/ml_method-glmnet/seed-0', 'dataset-otu_large/ml_method-rf/seed-0', 'dataset-otu_large/ml_method-glmnet/seed-1', 'dataset-otu_large/ml_method-rf/seed-1', 'dataset-otu_large/ml_method-glmnet/seed-2', 'dataset-otu_large/ml_method-rf/seed-2', 'dataset-otu_large/ml_method-glmnet/seed-3', 'dataset-otu_large/ml_method-rf/seed-3'] - + assert p.wildcard_pattern == "dataset-{dataset}/ml_method-{ml_method}/seed-{seed}" + assert list(p.instance_patterns) == [ + "dataset-otu_large/ml_method-glmnet/seed-0", + "dataset-otu_large/ml_method-rf/seed-0", + "dataset-otu_large/ml_method-glmnet/seed-1", + "dataset-otu_large/ml_method-rf/seed-1", + "dataset-otu_large/ml_method-glmnet/seed-2", + "dataset-otu_large/ml_method-rf/seed-2", + "dataset-otu_large/ml_method-glmnet/seed-3", + "dataset-otu_large/ml_method-rf/seed-3", + ] + + def test_set_default(): - assert set_default(config, 'dataset', None) == 'otu_large' - assert set_default(config, 'not_in_dict', 'value') == 'value' \ No newline at end of file + assert set_default(config, "dataset", None) == "otu_large" + assert set_default(config, "not_in_dict", "value") == "value" From 773e3f60335e52121b9e95a239eb4e2be15526af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Jan 2023 00:47:33 +0000 Subject: [PATCH 22/53] =?UTF-8?q?=F0=9F=90=B3=20Update=20Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4ecdc93..ab89436 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM condaforge/mambaforge:latest LABEL io.github.snakemake.containerized="true" -LABEL io.github.snakemake.conda_env_hash="d6e12a7cc311122c424028af8b7df4211faef9b3f3bc94298fd01affb1243862" +LABEL io.github.snakemake.conda_env_hash="20f54ad0e9c9ccd8cf96d25f486f156c95dd4d315b8b86da819213847f1e2e25" # Step 1: Retrieve conda environments @@ -41,21 +41,22 @@ COPY workflow/envs/mikropml.yml /conda-envs/67570867c99c9c3db185b41548ad6071/env # Conda environment: # source: workflow/envs/smk.yml -# prefix: /conda-envs/457b7b75191d44b96e5086432876e333 +# prefix: /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 # name: smk # channels: # - conda-forge # - bioconda # dependencies: +# - pandas # - snakemake=7 # - snakedeploy # - zip -RUN mkdir -p /conda-envs/457b7b75191d44b96e5086432876e333 -COPY workflow/envs/smk.yml /conda-envs/457b7b75191d44b96e5086432876e333/environment.yaml +RUN mkdir -p /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 +COPY workflow/envs/smk.yml /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1/environment.yaml # Step 2: Generate conda environments RUN mamba env create --prefix /conda-envs/b42323b0ffd5d034544511c9db1bdead --file /conda-envs/b42323b0ffd5d034544511c9db1bdead/environment.yaml && \ mamba env create --prefix /conda-envs/67570867c99c9c3db185b41548ad6071 --file /conda-envs/67570867c99c9c3db185b41548ad6071/environment.yaml && \ - mamba env create --prefix /conda-envs/457b7b75191d44b96e5086432876e333 --file /conda-envs/457b7b75191d44b96e5086432876e333/environment.yaml && \ + mamba env create --prefix /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 --file /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1/environment.yaml && \ mamba clean --all -y From 791473a6c58180d88235f327b14d1d7699d0dd61 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:09:31 -0500 Subject: [PATCH 23/53] Ignore results & figures from test runs --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a7dee2d..15d53db 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,7 @@ results/*/runs !.tests/ __pycache__/ .DS_Store -figures/otu* -results/otu* +figures/dataset* +results/dataset* report_otu* *.zip From 09b75b44ad513737c9593c7cde324c33751e10bb Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:09:53 -0500 Subject: [PATCH 24/53] Switch 'ml_method' to 'method' for compatibility in run_ml() --- config/config.yaml | 2 +- config/glmnet-rf.yaml | 2 +- config/robust.yaml | 2 +- config/test.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index e8185a7..aa67d0e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ dataset: otu_large outcome_colname: dx -ml_method: +method: - glmnet - rf kfold: 5 diff --git a/config/glmnet-rf.yaml b/config/glmnet-rf.yaml index 1c2e080..2c659af 100644 --- a/config/glmnet-rf.yaml +++ b/config/glmnet-rf.yaml @@ -1,7 +1,7 @@ dataset_csv: data/processed/otu-large.csv dataset_name: otu-large outcome_colname: dx -ml_methods: +methods: - glmnet - rf kfold: 5 diff --git a/config/robust.yaml b/config/robust.yaml index f4b3998..5411f5a 100644 --- a/config/robust.yaml +++ b/config/robust.yaml @@ -1,6 +1,6 @@ dataset: otu_large outcome_colname: dx -ml_method: +method: - glmnet - rf - rpart2 diff --git a/config/test.yaml b/config/test.yaml index 125a827..26f74a2 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -1,6 +1,6 @@ dataset: otu_micro outcome_colname: dx -ml_method: +method: - glmnet kfold: 2 ncores: 4 From 5fcaf6962c3dd11114ee30a8d1528fd7c27e974a Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:14:43 -0500 Subject: [PATCH 25/53] No need to specify inner_join(by) Just use all columns that match --- workflow/scripts/find_feature_importance.R | 2 +- workflow/scripts/train_ml.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/find_feature_importance.R b/workflow/scripts/find_feature_importance.R index 656c5d8..3b6e58d 100644 --- a/workflow/scripts/find_feature_importance.R +++ b/workflow/scripts/find_feature_importance.R @@ -40,6 +40,6 @@ wildcards <- schtools::get_wildcards_tbl() readr::write_csv( feat_imp %>% - inner_join(wildcards, by = c("method", "seed")), + inner_join(wildcards), snakemake@output[["feat"]] ) diff --git a/workflow/scripts/train_ml.R b/workflow/scripts/train_ml.R index c0d659a..f35a6ee 100644 --- a/workflow/scripts/train_ml.R +++ b/workflow/scripts/train_ml.R @@ -22,7 +22,7 @@ wildcards <- schtools::get_wildcards_tbl() readr::write_csv( ml_results$performance %>% - inner_join(wildcards, by = c("method", "seed")), + inner_join(wildcards), snakemake@output[["perf"]] ) readr::write_csv(ml_results$test_data, snakemake@output[["test"]]) From 07f9380c5e4282b82d5906a19758a05b40fbb019 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:15:31 -0500 Subject: [PATCH 26/53] Renamed 'ml_method' to 'method' for compatibility in run_ml() --- workflow/rules/learn.smk | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 75e8e35..23eb4d3 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -33,8 +33,11 @@ rule run_ml: benchmark: f"benchmarks/{paramspace.wildcard_pattern}/run_ml.txt" params: - params=paramspace.instance, + outcome_colname=outcome_colname, hyperparams=hyperparams, + method="{method}", + seed="{seed}", + kfold="{kfold}", threads: ncores resources: mem_mb=MEM_PER_GB * 4, @@ -70,7 +73,7 @@ rule find_feature_importance: f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", params: outcome_colname=outcome_colname, - method="{ml_method}", + method="{method}", seed="{seed}", threads: ncores resources: From 90f7e8f7a9811b6f459121d8d8cc41b681144f72 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:15:45 -0500 Subject: [PATCH 27/53] Encode dataset wildcard in aggregated results filenames --- workflow/Snakefile | 5 +++-- workflow/rules/combine.smk | 6 +++--- workflow/rules/plot.smk | 28 ++++++++++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index b4a236a..0a02116 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -37,7 +37,7 @@ rule targets: rule render_report: input: perf_plot=rules.plot_performance.output.plot, - feat_plot="figures/feature_importance.png", + feat_plot="figures/dataset-{dataset}/feature_importance.png", hp_plot=expand( "figures/{params}/hp_performance.png", params = instances_drop_wildcard(paramspace, 'seed') @@ -68,8 +68,9 @@ rule archive: expand(rules.render_report.input, dataset = dataset), expand(rules.render_report.output, dataset = dataset), expand( - "results/{rtype}-results.csv", + "results/dataset-{dataset}/{rtype}-results.csv", rtype=["performance", "feature_importance", "benchmarks", "sensspec"], + dataset = dataset ), output: f"workflow_{dataset}.zip", diff --git a/workflow/rules/combine.smk b/workflow/rules/combine.smk index 5816ab8..733806b 100644 --- a/workflow/rules/combine.smk +++ b/workflow/rules/combine.smk @@ -7,11 +7,11 @@ rule combine_results: params=paramspace.instance_patterns ), output: - csv="results/{type}-results.csv", + csv="results/dataset-{dataset}/{type}-results.csv", log: - "log/combine_results_{type}.txt", + "log/dataset-{dataset}/combine_results_{type}.txt", benchmark: - "benchmarks/combine_results_{type}.txt" + "benchmarks/dataset-{dataset}/combine_results_{type}.txt" conda: "../envs/mikropml.yml" script: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 40135e6..209a926 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -3,15 +3,15 @@ rule plot_performance: input: - csv="results/performance-results.csv", + csv="results/dataset-{dataset}/performance-results.csv", output: plot=report( - "figures/performance.png", + "figures/dataset-{dataset}/performance.png", category="Performance", subcategory="Model Performance", ), log: - "log/plot_performance.txt", + "log/dataset-{dataset}/plot_performance.txt", conda: "../envs/mikropml.yml" script: @@ -22,16 +22,16 @@ if find_feature_importance: rule plot_feature_importance: input: - csv="results/feature_importance-results.csv", + csv="results/dataset-{dataset}/feature_importance-results.csv", output: plot=report( - "figures/feature_importance.png", + "figures/dataset-{dataset}/feature_importance.png", category="Feature Importance", ), params: top_n=5, log: - "log/plot_feature_importance.txt", + "log/dataset-{dataset}/plot_feature_importance.txt", conda: "../envs/mikropml.yml" script: @@ -41,9 +41,9 @@ else: rule make_blank_feature_plot: output: - plot="figures/feature_importance.png", + plot="figures/dataset-{dataset}/feature_importance.png", log: - "log/make_blank_plot.txt", + "log/dataset-{dataset}/make_blank_plot.txt", conda: "../envs/mikropml.yml" script: @@ -69,16 +69,16 @@ rule plot_hp_performance: rule plot_benchmarks: input: - csv="results/benchmarks-results.csv", + csv="results/dataset-{dataset}/benchmarks-results.csv", output: plot=report( - "figures/benchmarks.png", + "figures/dataset-{dataset}/benchmarks.png", category="Performance", subcategory="Runtime & Memory Usage", caption="../report/benchmarks.rst", ), log: - "log/plot_benchmarks.txt", + "log/dataset-{dataset}/plot_benchmarks.txt", conda: "../envs/mikropml.yml" script: @@ -87,11 +87,11 @@ rule plot_benchmarks: rule plot_roc_curves: input: - csv="results/sensspec-results.csv", + csv="results/dataset-{dataset}/sensspec-results.csv", output: - plot="figures/roc_curves.png", + plot="figures/dataset-{dataset}/roc_curves.png", log: - "log/plot_roc_curves.txt", + "log/dataset-{dataset}/plot_roc_curves.txt", conda: "../envs/mikropml.yml" script: From 7e60ea100b8b6c9620d073e7f6a12bb1edf8bf28 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:18:27 -0500 Subject: [PATCH 28/53] Fix filenames for example report --- workflow/Snakefile | 4 +--- workflow/rules/example-report.smk | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 2b154e9..bf0a245 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -27,9 +27,7 @@ include: "rules/config.smk" include: "rules/learn.smk" include: "rules/combine.smk" include: "rules/plot.smk" - - -# include: "rules/example-report.smk" +include: "rules/example-report.smk" report: "report/workflow.rst" diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index dca8d59..ac00743 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -4,11 +4,14 @@ rule copy_example_figures: input: - figs=[ + figs=[ rules.plot_performance.output.plot, - rules.plot_feature_importance.output.plot, - expand(rules.plot_hp_performance.output.plot, ml_method=ml_methods), + "figures/dataset-{dataset}/feature_importance.png", rules.plot_benchmarks.output.plot, + expand( + "figures/{params}/hp_performance.png", + params=instances_drop_wildcard(paramspace, "seed"), + ), rules.plot_roc_curves.output.plot, "figures/graphviz/rulegraph.png", ], From 6842ef90dca780aea71b78455eba8b43e344b4d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Jan 2023 01:21:13 +0000 Subject: [PATCH 29/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/Snakefile | 2 +- workflow/rules/example-report.smk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index bf0a245..45195c2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -74,7 +74,7 @@ rule archive: expand( "results/dataset-{dataset}/{rtype}-results.csv", rtype=["performance", "feature_importance", "benchmarks", "sensspec"], - dataset = dataset + dataset=dataset, ), output: f"workflow_{dataset}.zip", diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index ac00743..6dabd87 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -4,7 +4,7 @@ rule copy_example_figures: input: - figs=[ + figs=[ rules.plot_performance.output.plot, "figures/dataset-{dataset}/feature_importance.png", rules.plot_benchmarks.output.plot, From 2c28be04629310a8d0191ea1b498b0e2a0e72453 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:24:00 -0500 Subject: [PATCH 30/53] Get method,seed,kfold from wildcards not params Because of this error with snakedeploy: WildcardError in file https://github.com/SchlossLab/mikropml-snakemake-workflow/raw/paramspace/workflow/rules/learn.smk, line 25: Wildcards in params cannot be determined from output files. Note that you have to use a function to deactivate automatic wildcard expansion in params strings, e.g., `lambda wildcards: '{test}'`. Also see https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#non-file-parameters-for-rules: 'method' --- workflow/rules/learn.smk | 2 -- workflow/scripts/find_feature_importance.R | 4 ++-- workflow/scripts/train_ml.R | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 5397276..2546ef8 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -74,8 +74,6 @@ rule find_feature_importance: f"log/{paramspace.wildcard_pattern}/find_feature_importance.txt", params: outcome_colname=outcome_colname, - method="{method}", - seed="{seed}", threads: ncores resources: mem_mb=MEM_PER_GB * 1, diff --git a/workflow/scripts/find_feature_importance.R b/workflow/scripts/find_feature_importance.R index 3b6e58d..1123b91 100644 --- a/workflow/scripts/find_feature_importance.R +++ b/workflow/scripts/find_feature_importance.R @@ -11,8 +11,8 @@ outcome_colname <- snakemake@params[["outcome_colname"]] train_dat <- model$trainingData names(train_dat)[names(train_dat) == ".outcome"] <- outcome_colname test_dat <- read_csv(snakemake@input[["test"]]) -method <- snakemake@params[["method"]] -seed <- as.numeric(snakemake@params[["seed"]]) +method <- snakemake@wildcards[["method"]] +seed <- as.numeric(snakemake@wildcards[["seed"]]) outcome_type <- get_outcome_type(c( train_dat %>% pull(outcome_colname), diff --git a/workflow/scripts/train_ml.R b/workflow/scripts/train_ml.R index f35a6ee..88e1628 100644 --- a/workflow/scripts/train_ml.R +++ b/workflow/scripts/train_ml.R @@ -3,27 +3,27 @@ library(dplyr) doFuture::registerDoFuture() future::plan(future::multicore, workers = snakemake@threads) -method <- snakemake@params[["method"]] -seed <- as.numeric(snakemake@params[["seed"]]) +method <- snakemake@wildcards[["method"]] +seed <- as.numeric(snakemake@wildcards[["seed"]]) +kfold <- as.numeric(snakemake@wildcards[["kfold"]]) hyperparams <- snakemake@params[["hyperparams"]][[method]] +outcome_colname <- snakemake@params[["outcome_colname"]] data_processed <- readRDS(snakemake@input[["rds"]])$dat_transformed ml_results <- mikropml::run_ml( dataset = data_processed, method = method, - outcome_colname = snakemake@params[["outcome_colname"]], + outcome_colname = outcome_colname, find_feature_importance = FALSE, - kfold = as.numeric(snakemake@params[["kfold"]]), + kfold = kfold, seed = seed, hyperparameters = hyperparams ) wildcards <- schtools::get_wildcards_tbl() -readr::write_csv( - ml_results$performance %>% - inner_join(wildcards), - snakemake@output[["perf"]] -) -readr::write_csv(ml_results$test_data, snakemake@output[["test"]]) -saveRDS(ml_results$trained_model, file = snakemake@output[["model"]]) +ml_results$performance %>% + inner_join(wildcards) %>% + readr::write_csv(snakemake@output[["perf"]]) +ml_results$test_data %>% readr::write_csv(snakemake@output[["test"]]) +ml_results$trained_model %>% saveRDS(file = snakemake@output[["model"]]) From 0c8629a1fec183c681d02b76fd520e5eed808f98 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:26:12 -0500 Subject: [PATCH 31/53] Remove params that are redundant with wildcards --- workflow/rules/learn.smk | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflow/rules/learn.smk b/workflow/rules/learn.smk index 2546ef8..2106813 100644 --- a/workflow/rules/learn.smk +++ b/workflow/rules/learn.smk @@ -36,9 +36,6 @@ rule run_ml: params: outcome_colname=outcome_colname, hyperparams=hyperparams, - method="{method}", - seed="{seed}", - kfold="{kfold}", threads: ncores resources: mem_mb=MEM_PER_GB * 4, From e05e36c9557dda4cea961a5ad366415c46b0de4c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:30:23 -0500 Subject: [PATCH 32/53] Silence snakemake linter warning about abs paths These are not absolute paths, but the snakemake linter thinks they are. Moving the slash to the end of the regex should fix it. Lints for snakefile /github/workspace/workflow/scripts/functions.py: * Absolute path "/{{0,1}}{wildcard}{paramspace.param_sep}{{{wildcard}}}" in line 36: Do not define absolute paths inside of the workflow, since this renders your workflow irreproducible on other machines. Use path relative to the working directory instead, or make the path configurable via a config file. Also see: https://snakemake.readthedocs.io/en/latest/snakefiles/configuration.html#configuration * Absolute path "/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*" in line 54: Do not define absolute paths inside of the workflow, since this renders your workflow irreproducible on other machines. Use path relative to the working directory instead, or make the path configurable via a config file. Also see: https://snakemake.readthedocs.io/en/latest/snakefiles/configuration.html#configuration --- workflow/scripts/functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/functions.py b/workflow/scripts/functions.py index 8a7326d..c825a62 100644 --- a/workflow/scripts/functions.py +++ b/workflow/scripts/functions.py @@ -33,7 +33,7 @@ def pattern_drop_wildcard(paramspace, wildcard): :rtype: str """ return re.sub( - f"/{{0,1}}{wildcard}{paramspace.param_sep}{{{wildcard}}}", + f"{wildcard}{paramspace.param_sep}{{{wildcard}}}/{{0,1}}", "", paramspace.wildcard_pattern, ).strip("/") @@ -51,7 +51,7 @@ def instances_drop_wildcard(paramspace, wildcard): :rtype: set """ return { - re.sub(f"/{{0,1}}{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*", "", i).strip( + re.sub(f"{wildcard}{paramspace.param_sep}[a-zA-Z_0-9]*/{{0,1}}", "", i).strip( "/" ) for i in paramspace.instance_patterns From a1f3f89e53e183bb6a3e40c808b5bc0320f6b6a6 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 20:43:43 -0500 Subject: [PATCH 33/53] New rule to write the paramspace to csv This may be needed because workflows that use this module with snakedeploy won't have access to the paramspace object --- workflow/Snakefile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index 45195c2..fb47864 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -67,6 +67,18 @@ rule render_report: "scripts/report.Rmd" +rule write_paramspace: + output: + csv='config/paramspace.csv' + log: + 'log/write_paramspace.txt' + params: + paramspace=paramspace + run: + print(params.paramspace.wildcard_pattern) + params.paramspace.dataframe.to_csv(output.csv, index=False) + + rule archive: input: expand(rules.render_report.input, dataset=dataset), @@ -76,6 +88,7 @@ rule archive: rtype=["performance", "feature_importance", "benchmarks", "sensspec"], dataset=dataset, ), + rules.write_paramspace.output output: f"workflow_{dataset}.zip", log: From fa09085219371143ff37676095928f7cf98306aa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Jan 2023 01:46:37 +0000 Subject: [PATCH 34/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/Snakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index fb47864..c870896 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -69,11 +69,11 @@ rule render_report: rule write_paramspace: output: - csv='config/paramspace.csv' + csv="config/paramspace.csv", log: - 'log/write_paramspace.txt' + "log/write_paramspace.txt", params: - paramspace=paramspace + paramspace=paramspace, run: print(params.paramspace.wildcard_pattern) params.paramspace.dataframe.to_csv(output.csv, index=False) @@ -88,7 +88,7 @@ rule archive: rtype=["performance", "feature_importance", "benchmarks", "sensspec"], dataset=dataset, ), - rules.write_paramspace.output + rules.write_paramspace.output, output: f"workflow_{dataset}.zip", log: From 3ac6da841722e93fac3bfc759b41165aea64fd7c Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 21:37:05 -0500 Subject: [PATCH 35/53] Move write_paramspace code to script to satisfy linter Nothing was actually wrong with it. Snakemake has a bug (snakemake issue #2035). --- workflow/Snakefile | 11 ++++++----- workflow/scripts/write_paramspace.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 workflow/scripts/write_paramspace.py diff --git a/workflow/Snakefile b/workflow/Snakefile index fb47864..ce73256 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -67,17 +67,18 @@ rule render_report: "scripts/report.Rmd" +# TODO: switch this back to a `run` directive when snakemake fixes https://github.com/snakemake/snakemake/issues/2035 rule write_paramspace: output: csv='config/paramspace.csv' log: 'log/write_paramspace.txt' params: - paramspace=paramspace - run: - print(params.paramspace.wildcard_pattern) - params.paramspace.dataframe.to_csv(output.csv, index=False) - + df=paramspace.dataframe + conda: 'envs/smk.yml' + script: + 'scripts/write_paramspace.py' + rule archive: input: diff --git a/workflow/scripts/write_paramspace.py b/workflow/scripts/write_paramspace.py new file mode 100644 index 0000000..2be948d --- /dev/null +++ b/workflow/scripts/write_paramspace.py @@ -0,0 +1 @@ +snakemake.params.df.to_csv(snakemake.output.csv, index=False) From 79bfce418d9a3420ed599efb92504cf5f881228d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 29 Jan 2023 02:40:55 +0000 Subject: [PATCH 36/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/Snakefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 847d97e..1ec9701 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -74,11 +74,12 @@ rule write_paramspace: log: "log/write_paramspace.txt", params: - df=paramspace.dataframe - conda: 'envs/smk.yml' + df=paramspace.dataframe, + conda: + "envs/smk.yml" script: - 'scripts/write_paramspace.py' - + "scripts/write_paramspace.py" + rule archive: input: From d688ae3d86fd2758d8d273aee3921587bd5265ea Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sat, 28 Jan 2023 22:16:18 -0500 Subject: [PATCH 37/53] Include shell magic for conda env --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0fa4b43..8f890f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -44,5 +44,6 @@ jobs: # run: | # pytest --workers 2 .tests/ - name: Test with pytest + shell: bash -el {0} run: | pytest --workers 2 workflow/scripts/ \ No newline at end of file From 5ffec5bf4a6747f39bfa4c297aeb067d2c2c5db3 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Sun, 29 Jan 2023 21:25:52 -0500 Subject: [PATCH 38/53] Switch from pytest-parallel to pytest-xdist Got a warning in pytest-parallel, found out it's no longer maintained. https://github.com/kevlened/pytest-parallel/issues/118 --- .github/workflows/tests.yml | 4 ++-- workflow/envs/github-actions.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8f890f3..12c380f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,8 +42,8 @@ jobs: args: "archive --forceall --cores 2 --use-conda --conda-frontend mamba --conda-cleanup-pkgs cache --show-failed-logs --all-temp --configfile config/test.yaml" # - name: Test with pytest # run: | -# pytest --workers 2 .tests/ +# pytest -n 2 .tests/ - name: Test with pytest shell: bash -el {0} run: | - pytest --workers 2 workflow/scripts/ \ No newline at end of file + pytest -n 2 workflow/scripts/ \ No newline at end of file diff --git a/workflow/envs/github-actions.yml b/workflow/envs/github-actions.yml index 266b249..9f9e175 100644 --- a/workflow/envs/github-actions.yml +++ b/workflow/envs/github-actions.yml @@ -7,7 +7,7 @@ channels: dependencies: - black - pytest - - pytest-parallel + - pytest-xdist - python=3.11 - r-base=4 - r-styler From 2262d628c368f1c6b6c9dd0fda481cfd2df28f8d Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Jan 2023 15:57:13 -0500 Subject: [PATCH 39/53] Use smk env for test action --- .github/workflows/tests.yml | 4 ++-- workflow/envs/github-actions.yml | 2 -- workflow/envs/smk.yml | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12c380f..302baa2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,8 +26,8 @@ jobs: python-version: 3.11 miniforge-variant: Mambaforge miniforge-version: latest - activate-environment: github-actions - environment-file: workflow/envs/github-actions.yml + activate-environment: smk + environment-file: workflow/envs/smk.yml - name: Lint workflow uses: snakemake/snakemake-github-action@v1.24.0 with: diff --git a/workflow/envs/github-actions.yml b/workflow/envs/github-actions.yml index 9f9e175..0ad26a9 100644 --- a/workflow/envs/github-actions.yml +++ b/workflow/envs/github-actions.yml @@ -6,8 +6,6 @@ channels: - r dependencies: - black - - pytest - - pytest-xdist - python=3.11 - r-base=4 - r-styler diff --git a/workflow/envs/smk.yml b/workflow/envs/smk.yml index f498265..6cf2a83 100644 --- a/workflow/envs/smk.yml +++ b/workflow/envs/smk.yml @@ -4,6 +4,8 @@ channels: - bioconda dependencies: - pandas + - pytest + - pytest-xdist - snakemake=7 - snakedeploy - zip From a72de4a709a63d70aabbc62b8e2402bb5bf05442 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 21:01:22 +0000 Subject: [PATCH 40/53] =?UTF-8?q?=F0=9F=90=B3=20Update=20Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index ab89436..b4e34c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM condaforge/mambaforge:latest LABEL io.github.snakemake.containerized="true" -LABEL io.github.snakemake.conda_env_hash="20f54ad0e9c9ccd8cf96d25f486f156c95dd4d315b8b86da819213847f1e2e25" +LABEL io.github.snakemake.conda_env_hash="fd5d5c6702a929d9f0d6140ffbb1403df6356c3566104c834ff13710987d18f4" # Step 1: Retrieve conda environments @@ -41,22 +41,24 @@ COPY workflow/envs/mikropml.yml /conda-envs/67570867c99c9c3db185b41548ad6071/env # Conda environment: # source: workflow/envs/smk.yml -# prefix: /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 +# prefix: /conda-envs/bbc262640c3353e62cad877627dd3174 # name: smk # channels: # - conda-forge # - bioconda # dependencies: # - pandas +# - pytest +# - pytest-xdist # - snakemake=7 # - snakedeploy # - zip -RUN mkdir -p /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 -COPY workflow/envs/smk.yml /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1/environment.yaml +RUN mkdir -p /conda-envs/bbc262640c3353e62cad877627dd3174 +COPY workflow/envs/smk.yml /conda-envs/bbc262640c3353e62cad877627dd3174/environment.yaml # Step 2: Generate conda environments RUN mamba env create --prefix /conda-envs/b42323b0ffd5d034544511c9db1bdead --file /conda-envs/b42323b0ffd5d034544511c9db1bdead/environment.yaml && \ mamba env create --prefix /conda-envs/67570867c99c9c3db185b41548ad6071 --file /conda-envs/67570867c99c9c3db185b41548ad6071/environment.yaml && \ - mamba env create --prefix /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1 --file /conda-envs/9bcbdde863b9fd5392599c9ca47b5cc1/environment.yaml && \ + mamba env create --prefix /conda-envs/bbc262640c3353e62cad877627dd3174 --file /conda-envs/bbc262640c3353e62cad877627dd3174/environment.yaml && \ mamba clean --all -y From 48c7b3560baad18db64855ab5f15677e12fede5a Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Jan 2023 17:19:30 -0500 Subject: [PATCH 41/53] Note pandas dependency --- config/README.md | 8 +++++++- quick-start.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/README.md b/config/README.md index 185b1a6..6336f7a 100644 --- a/config/README.md +++ b/config/README.md @@ -1,4 +1,10 @@ -# General configuration +# Additional Dependencies + +Besides snakemake, you will also need `pandas` to run this workflow: + +`mamba install pandas` + +# General Configuration To configure this workflow, modify [`config/config.yaml`](/config/config.yaml) according to your needs. diff --git a/quick-start.md b/quick-start.md index fa94fb1..a7b9078 100644 --- a/quick-start.md +++ b/quick-start.md @@ -18,7 +18,7 @@ 1. If you don't have conda/mamba yet, we recommend installing [Mambaforge](https://mamba.readthedocs.io/en/latest/installation.html). - 1. Create a conda environment with snakemake installed: + 1. Create a conda environment with snakemake and pandas installed: ``` sh mamba env create -f workflow/envs/smk.yml From 7df13d3114d546aaa357d276ebcec83c370edce4 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Jan 2023 17:36:33 -0500 Subject: [PATCH 42/53] Fill in dataset wildcard for example report --- workflow/rules/example-report.smk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index 6dabd87..fcd0615 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -5,24 +5,24 @@ rule copy_example_figures: input: figs=[ - rules.plot_performance.output.plot, - "figures/dataset-{dataset}/feature_importance.png", - rules.plot_benchmarks.output.plot, + f"figures/dataset-{dataset}/performance.png", + f"figures/dataset-{dataset}/feature_importance.png", + f"figures/dataset-{dataset}/benchmarks.png", + f"figures/dataset-{dataset}/roc_curves.png", expand( "figures/{params}/hp_performance.png", params=instances_drop_wildcard(paramspace, "seed"), ), - rules.plot_roc_curves.output.plot, "figures/graphviz/rulegraph.png", ], output: perf_plot="figures/example/performance.png", feat_plot="figures/example/feature_importance.png", bench_plot="figures/example/benchmarks.png", + roc_plot="figures/example/roc_curves.png", hp_plot=expand( "figures/example/hp_performance_{ml_method}.png", ml_method=ml_methods ), - roc_plot="figures/example/roc_curves.png", rulegraph="figures/example/rulegraph.png", log: "log/copy_example_figures.txt", @@ -42,9 +42,9 @@ rule make_example_report: input: perf_plot=rules.copy_example_figures.output.perf_plot, feat_plot=rules.copy_example_figures.output.feat_plot, - hp_plot=rules.copy_example_figures.output.hp_plot, bench_plot=rules.copy_example_figures.output.bench_plot, roc_plot=rules.copy_example_figures.output.roc_plot, + hp_plot=rules.copy_example_figures.output.hp_plot, rulegraph=rules.copy_example_figures.output.rulegraph, output: doc="report-example.md", From 2333b1efa791e2234fa664c4dcb27c04d9545c96 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Jan 2023 17:57:45 -0500 Subject: [PATCH 43/53] Add option to use a custom paramspace csv file --- config/README.md | 6 +++++- config/config.yaml | 2 ++ config/custom-paramspace.csv | 11 +++++++++++ config/custom-paramspace.yaml | 19 +++++++++++++++++++ config/glmnet-rf.yaml | 1 + config/robust.yaml | 1 + config/test.yaml | 1 + workflow/rules/config.smk | 4 ++-- 8 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 config/custom-paramspace.csv create mode 100644 config/custom-paramspace.yaml diff --git a/config/README.md b/config/README.md index 6336f7a..77d8509 100644 --- a/config/README.md +++ b/config/README.md @@ -33,9 +33,13 @@ according to your needs. for each ML method (optional). Leave this blank if you'd like to use the defaults. You will have to set these if you wish to use an ML method from caret that we don't officially support. + - `paramspace_csv`: if you'd like to use a custom csv file to build the + paramspace, specify the path to the csv file here. If `None`, then the + paramspace will be built based on the parameters in the configfile. - `exclude_param_keys`: keys in the configfile to exclude from the parameter space. All keys in the configfile not listed in `exclude_param_keys` will be - included as wildcards for `run_ml` and other rules. + included as wildcards for `run_ml` and other rules. This option is ignored + if `paramspace_csv` is not `None`. We also provide [`config/test.yaml`](/config/test.yaml), which uses a smaller dataset so you can first make sure the workflow runs without error on your diff --git a/config/config.yaml b/config/config.yaml index aa67d0e..8f1c3bd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -8,6 +8,7 @@ ncores: 8 nseeds: 10 find_feature_importance: true hyperparams: +paramspace_csv: exclude_param_keys: - exclude_param_keys - outcome_colname @@ -15,3 +16,4 @@ exclude_param_keys: - nseeds - find_feature_importance - hyperparams + - paramspace_csv diff --git a/config/custom-paramspace.csv b/config/custom-paramspace.csv new file mode 100644 index 0000000..1d003b3 --- /dev/null +++ b/config/custom-paramspace.csv @@ -0,0 +1,11 @@ +dataset,kfold,method,seed +otu_large,5,glmnet,100 +otu_large,5,glmnet,101 +otu_large,5,glmnet,102 +otu_large,5,glmnet,103 +otu_large,5,glmnet,104 +otu_large,5,rf,105 +otu_large,5,rf,106 +otu_large,5,rf,107 +otu_large,5,rf,108 +otu_large,5,rf,109 diff --git a/config/custom-paramspace.yaml b/config/custom-paramspace.yaml new file mode 100644 index 0000000..d90b08a --- /dev/null +++ b/config/custom-paramspace.yaml @@ -0,0 +1,19 @@ +dataset: otu_large +outcome_colname: dx +method: + - glmnet + - rf +kfold: 5 +ncores: 8 +nseeds: 10 +find_feature_importance: true +hyperparams: +paramspace_csv: 'config/custom-paramspace.csv' +exclude_param_keys: + - exclude_param_keys + - outcome_colname + - ncores + - nseeds + - find_feature_importance + - hyperparams + - paramspace_csv diff --git a/config/glmnet-rf.yaml b/config/glmnet-rf.yaml index 2c659af..00d4373 100644 --- a/config/glmnet-rf.yaml +++ b/config/glmnet-rf.yaml @@ -15,3 +15,4 @@ exclude_param_keys: - nseeds - find_feature_importance - hyperparams + - paramspace_csv diff --git a/config/robust.yaml b/config/robust.yaml index 5411f5a..84063ab 100644 --- a/config/robust.yaml +++ b/config/robust.yaml @@ -32,3 +32,4 @@ exclude_param_keys: - nseeds - find_feature_importance - hyperparams + - paramspace_csv diff --git a/config/test.yaml b/config/test.yaml index 26f74a2..13256d2 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -24,3 +24,4 @@ exclude_param_keys: - nseeds - find_feature_importance - hyperparams + - paramspace_csv diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index ef7f1d3..652b792 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -20,8 +20,8 @@ start_seed = 100 seeds = range(start_seed, start_seed + nseeds) config["seed"] = list(seeds) -# parameter space based on configfile -paramspace = get_paramspace_from_config(config) +# create parameter space based on custom csv or configfile +paramspace = Paramspace(pd.read_csv(config['paramspace_csv']), param_sep="-") if 'paramspace_csv' in config and config['paramspace_csv'] else get_paramspace_from_config(config) # wildcard pattern without seed. needed for rule `combine_hp_performance` wildcard_no_seed = pattern_drop_wildcard(paramspace, "seed") # wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` From 67c3d4a8f5621cb8f4e3e72714f8673ce5f9fb2f Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 30 Jan 2023 18:01:39 -0500 Subject: [PATCH 44/53] Link to snakemake docs on paramspace --- config/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/README.md b/config/README.md index 77d8509..4475811 100644 --- a/config/README.md +++ b/config/README.md @@ -34,7 +34,7 @@ according to your needs. defaults. You will have to set these if you wish to use an ML method from caret that we don't officially support. - `paramspace_csv`: if you'd like to use a custom csv file to build the - paramspace, specify the path to the csv file here. If `None`, then the + [paramspace](https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#parameter-space-exploration) for `run_ml`, specify the path to the csv file here. If `None`, then the paramspace will be built based on the parameters in the configfile. - `exclude_param_keys`: keys in the configfile to exclude from the parameter space. All keys in the configfile not listed in `exclude_param_keys` will be From 78b6c12f9bd0f03cff1cec7bf233d6221559b6dc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 23:04:23 +0000 Subject: [PATCH 45/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/rules/config.smk | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 652b792..4ee0baa 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -21,7 +21,11 @@ seeds = range(start_seed, start_seed + nseeds) config["seed"] = list(seeds) # create parameter space based on custom csv or configfile -paramspace = Paramspace(pd.read_csv(config['paramspace_csv']), param_sep="-") if 'paramspace_csv' in config and config['paramspace_csv'] else get_paramspace_from_config(config) +paramspace = ( + Paramspace(pd.read_csv(config["paramspace_csv"]), param_sep="-") + if "paramspace_csv" in config and config["paramspace_csv"] + else get_paramspace_from_config(config) +) # wildcard pattern without seed. needed for rule `combine_hp_performance` wildcard_no_seed = pattern_drop_wildcard(paramspace, "seed") # wildcard pattern with all wildcards _except_ seed having double curly braces for use by `expand()` in rule `combine_hp_performance` From 8a0d3467d290da17523c556953a2bbf6ff16ff14 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 31 Jan 2023 15:25:31 -0500 Subject: [PATCH 46/53] Tweak medium config --- config/{glmnet-rf.yaml => glmnet.yaml} | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) rename config/{glmnet-rf.yaml => glmnet.yaml} (72%) diff --git a/config/glmnet-rf.yaml b/config/glmnet.yaml similarity index 72% rename from config/glmnet-rf.yaml rename to config/glmnet.yaml index 2c659af..0991243 100644 --- a/config/glmnet-rf.yaml +++ b/config/glmnet.yaml @@ -1,9 +1,7 @@ -dataset_csv: data/processed/otu-large.csv -dataset_name: otu-large +dataset: otu_large outcome_colname: dx -methods: +method: - glmnet - - rf kfold: 5 ncores: 16 nseeds: 50 From 911a343ad123cc4919bb095d4ef171e9f2755783 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Tue, 31 Jan 2023 22:46:32 -0500 Subject: [PATCH 47/53] Fix typo --- workflow/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 8b56f56..cfa1c75 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -35,7 +35,7 @@ include: "rules/example-report.smk" results_types = ["performance", "benchmarks", "sensspec"] if find_feature_importance: - results_types.append("feature-importance") + results_types.append("feature_importance") rule targets: From b22a22d631ddb128e7cf11b1381115e0143f070b Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 2 Feb 2023 11:35:45 -0500 Subject: [PATCH 48/53] Separate rule to copy hp plots for example report --- workflow/rules/example-report.smk | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index fcd0615..b3ccb5d 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -1,6 +1,19 @@ """ Create an example report """ +rule copy_hp_plot: + input: + plot=f"figures/{wildcard_no_seed}/hp_performance.png" + output: + plot=f"figures/example/{wildcard_no_seed}/hp_performance.png" + log: + f"log/{wildcard_no_seed}/copy_hp_plot.txt", + conda: + "../envs/smk.yml" + shell: + """ + cp {input.plot} {output.plot} &> {log} + """ rule copy_example_figures: input: @@ -9,10 +22,6 @@ rule copy_example_figures: f"figures/dataset-{dataset}/feature_importance.png", f"figures/dataset-{dataset}/benchmarks.png", f"figures/dataset-{dataset}/roc_curves.png", - expand( - "figures/{params}/hp_performance.png", - params=instances_drop_wildcard(paramspace, "seed"), - ), "figures/graphviz/rulegraph.png", ], output: @@ -20,9 +29,6 @@ rule copy_example_figures: feat_plot="figures/example/feature_importance.png", bench_plot="figures/example/benchmarks.png", roc_plot="figures/example/roc_curves.png", - hp_plot=expand( - "figures/example/hp_performance_{ml_method}.png", ml_method=ml_methods - ), rulegraph="figures/example/rulegraph.png", log: "log/copy_example_figures.txt", @@ -34,7 +40,7 @@ rule copy_example_figures: """ for f in {input.figs}; do cp $f {params.outdir} - done + done &> {log} """ @@ -44,8 +50,9 @@ rule make_example_report: feat_plot=rules.copy_example_figures.output.feat_plot, bench_plot=rules.copy_example_figures.output.bench_plot, roc_plot=rules.copy_example_figures.output.roc_plot, - hp_plot=rules.copy_example_figures.output.hp_plot, rulegraph=rules.copy_example_figures.output.rulegraph, + hp_plot=expand("figures/example/{params}/hp_performance.png", + params = instances_drop_wildcard(paramspace, "seed")) output: doc="report-example.md", log: From e202855536cae37d162bea30c8dead1de2d93d83 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Feb 2023 16:39:24 +0000 Subject: [PATCH 49/53] =?UTF-8?q?=F0=9F=8E=A8=20=20Style=20Python=20&=20Sn?= =?UTF-8?q?akemake=20code=20=F0=9F=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- workflow/rules/example-report.smk | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/workflow/rules/example-report.smk b/workflow/rules/example-report.smk index b3ccb5d..446c214 100644 --- a/workflow/rules/example-report.smk +++ b/workflow/rules/example-report.smk @@ -1,11 +1,12 @@ """ Create an example report """ + rule copy_hp_plot: input: - plot=f"figures/{wildcard_no_seed}/hp_performance.png" + plot=f"figures/{wildcard_no_seed}/hp_performance.png", output: - plot=f"figures/example/{wildcard_no_seed}/hp_performance.png" + plot=f"figures/example/{wildcard_no_seed}/hp_performance.png", log: f"log/{wildcard_no_seed}/copy_hp_plot.txt", conda: @@ -15,6 +16,7 @@ rule copy_hp_plot: cp {input.plot} {output.plot} &> {log} """ + rule copy_example_figures: input: figs=[ @@ -51,8 +53,10 @@ rule make_example_report: bench_plot=rules.copy_example_figures.output.bench_plot, roc_plot=rules.copy_example_figures.output.roc_plot, rulegraph=rules.copy_example_figures.output.rulegraph, - hp_plot=expand("figures/example/{params}/hp_performance.png", - params = instances_drop_wildcard(paramspace, "seed")) + hp_plot=expand( + "figures/example/{params}/hp_performance.png", + params=instances_drop_wildcard(paramspace, "seed"), + ), output: doc="report-example.md", log: From 4c6b5288758cad8e0cd8581f6d60bc0b11ab5836 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 2 Feb 2023 11:45:00 -0500 Subject: [PATCH 50/53] Fix ml_method->method key name for config defaults --- workflow/rules/config.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rules/config.smk b/workflow/rules/config.smk index 4ee0baa..16ffa36 100644 --- a/workflow/rules/config.smk +++ b/workflow/rules/config.smk @@ -9,7 +9,7 @@ config_path = ( dataset = config["dataset"] ncores = config["ncores"] -ml_methods = set_default(config, "ml_method", "glmnet") +ml_methods = set_default(config, "method", "glmnet") kfold = set_default(config, "kfold", 5) outcome_colname = set_default(config, "outcome_colname", None) hyperparams = set_default(config, "hyperparams", None) From c1ea2a89e056a247f3eccb224066662ae920cc53 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 2 Feb 2023 11:55:26 -0500 Subject: [PATCH 51/53] Update example report --- .../kfold-5/method-glmnet/hp_performance.png | Bin 0 -> 30678 bytes .../kfold-5/method-rf/hp_performance.png | Bin 0 -> 29775 bytes figures/example/hp_performance_glmnet.png | Bin 53143 -> 0 bytes figures/example/hp_performance_rf.png | Bin 48456 -> 0 bytes report-example.md | 6 +++--- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 figures/example/dataset-otu_large/kfold-5/method-glmnet/hp_performance.png create mode 100644 figures/example/dataset-otu_large/kfold-5/method-rf/hp_performance.png delete mode 100644 figures/example/hp_performance_glmnet.png delete mode 100644 figures/example/hp_performance_rf.png diff --git a/figures/example/dataset-otu_large/kfold-5/method-glmnet/hp_performance.png b/figures/example/dataset-otu_large/kfold-5/method-glmnet/hp_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..133167cca36140c0e7afc14fe5b51f15891da8b3 GIT binary patch literal 30678 zcmeFacU03!_c$DvU9bgER4_D=RcsU`G$~n^rXry0BE2g_T0nY@3Ivd*%Sw?FffW>_ z386!TAfO0{^cq6%p(l`p{3d`t&-a|~`<}Nv-}9E^A2}!a%*?%aZkv1W%zP4dQ%4iJ zAGRL?fk3ZaRr>=1*$ZcV_OgK|zE9L>5C|LO=8Zqq!OuN=_WbnIPi$;#KmYvmFTec4 z&d$!k!LfJm-hKP_?ccxuz<~n?4<3XzJ_^84?Sg@uKsrKOdXm9@3?-Me@1-MeRFV`FP;i$o&t z-@k8XXJ>D3@8ICz=;-L=l>gML=?(Xj4;o<4&>E-3+?d|R3 z*wd^@9!TF5P(LbA3l8e=+UFcj~@pH1_lKM1qTOXFqn{#kkHW3u&}TvPo9K_ zhet$2L`FtNMMXtNM?Zb~^x3m#fBp4WOiaw*fB*gb`SaM=*tod3`1tsQgoGC_UL+e_vBmQ(Ie0AQ0;6>OOq*!c0|$4{R=H8nMT{`|SQxw)mK zrM0#7%a<>0ZEat_er<1W@9600?Ck97>gw+9?&<02?d|RB>+A3D9~c-I92^`P8X^*j z!^6WQ5@}>)WOQ_NY;0_Ne0*YJVsdhFYHDhFdU|GNW_EUVZfZCe?)o8FxJ33BAMhVmYWg;FRx2Wp-L8@fDW2zd|F0**ml4R`~!?JtBG+GIT&;?ffi_J@{-q1j}zIvd3G z5z@dPEdqnYA5bIA_?9Cf782ah?0L=S-@YA4+w)MtaxRUuG~0@pX9!o##C?3^2Y*ZN zYKTT`S({_4GZ0=MG>4d}t|+7Dcna3Ur=F&>HP+etv%xxUMF-6^*GjDbq`20` zs}qztRMKDMlV;I9J*FK!MiuUd$PpMx*%JJ*QPBn!|K-(8fnv9j_m1JR@dDIlX`#gO z_VwPScL+n@vJnnRknQ7gp(xz zd?l#$-)0SL;U=8}TYu0$Hf8Ffx0vwy)_!N7VaYfg8Rx$Kq}$Vk+0At8W@9$Vnz}E} z7#oj*>6Tf)O(XHxyPKx#M&pT8W-C%~E&_wLcf#W5UM1V&t@P;hzGOC(6LEg5H(GM4 zd60rG~>(=^5o|1~i;`&yiKXFn*#kD((-GD|bSedhiiRLLRxtec&eukHE*+gmM z=Sg#6k1`p9r{sow7@6w^nPr7-Xl;SA$$SDQEplQ@Br=k)<>-axg-E%L9)b4DMKs_wX7rFEdR7xd$bICShe9e68RwgNql*6HEf1r#c)XrCsA4z=@m zdyV8Awk`2wX(EA7ov^1ld}X1X7F|_@f)B{M+?}01zj5F+`}&JMrs2x-C~d3BUVeu2 zW^OBrvoZ@Oy)^TI>0N1kx&%%%UaFW^W^rChrEQT~7b2cF%U4FwM>@m?_5)-&gG{w*1XVN0fLF?OhmucNG)-!C603e1PUUJNFY;3aL{{pFFg z2Ep}m3*Fgu*j4f;NV*6!>vx$DOJvjZJUB-bC@mPBLrj-XhUuE>Q-Tl%G0 zju3dTbFgxBJ+dcr^m7P zIhYZu=m-?@89K@XRg}wX=5>#DR0=}&6atBZ(LCF z-aN>|vpa}~e4nb;{iUjtJB^gw+)Qbfy0|Mhsio;KE)OW9K`MaL14EUvC@Qj{kxlow z2E5KzrX+73@eNtNUHHU@jV3`+9($RV0Qe0*!Uz5ePDK79kEiZDaLU`f{n)CqHNCn; zNr5LJmeh|65Urh4{{zQ*>GbuK#VE<1k;9=1o=1rr5GQ+~iM3hN5L}K~Z$_#5zH^wG9NK+?Rgnce-<`!bXHH$!}Q`?WT3fV0<^_BjC>bwlk22IRWtn!;dcFBO&)^eSIxi9XgxhfrSOh3p?bL(SJ zL2uSg?tx^Z-lmiI zn~^<^9fc5}tCeXa5|$zZGOaDGa&MVa2T%9#$95sQCo~ak%w`D5?f>>+_Jtss%`8*_El*Kb&vF@8j61E6PG0S=Cx3>xelSp1Rr1JL zKt1ev@bR@RcvA1Bu39<4b@?ySSrns}hxikOFXCc2hL1TKj~}v&@zx!B(9gzP+Pc;H z3uNN20${cl@PDn!-(nuWP{IQWf#~@NAk?Qaya~ZnVHaoTdYfSJTM7k4X zd;@4B0&`MW(GK?*qD(aje-tvsjg7JKF1%xg$=%>mey~AXH!W>X(M8Ld~Ha`l?NHBZn~?tpD}na!te*ebuM(rtr4(&eU*UaO9g;mkDAXvKe)tmBMh z2wV$9KE!a3D8+&9Mf((%nmrM+SbkfwbZSmcM$78nf(~$E5koK2EnW61nl2kykA`rB z-0@m< z<&eAsmK5L9a;^I;@p_Ti$>4nb3UM7M>ES@t6Ltt+)gDb%(LJ2@DD>GNxpe^*c#vd? ztgt+@JdC0&Bv^ZVo;lxRZGuqtrmue_iaEM z%=@2&o6iZ=KF?|H4q5CR7~GynjRrwoq|)Dqyqv`*gz1JKqg9l^g<0whD~I=mtB_tq$Qt=DNIBw={S!49vfnnE2G%%0=q< zNIJ^&`EpnK0=Ri1`>IXr|Mt20UTH&qlJiM}sX7YEvOFu!U%v7oeFt`(_(47UvflP&<0 z?yk5KwdJ-~(*r2@?Y(i@^ZqA;(yi#dbnXoN!IgY&m6$SPYzpLI8@~gbej-RP69I8~ zu-DdIP~{)ju*TLKm3LfBM~}6Wt*Op?$sY%Dz~oq6FAy&V^=GWFll`nxr|VjtP!0By z!Db2+zxGP~_d1Fndki;|z7nkcy=DEI70HcXR{7Smd>7=vw=Fj5MXyJ!E_64n=?BoS_d?$9ZJtL`KfN1j zTGTFFJQKL&8;K88Pfr1>RswSBGz{`Gx=a9yGR<}=;g^gTqW0P(I#%{hvTw-;f|QW2 z05o#Qk&AtrSKs?LGh8`#pI>Y{0fvSmxuLAw5;%Y}Ct+Yk1DpYF3na9FOX_NYfN3Yw zuE1H$`qBLdXMQNb5A*op34Zve=oEyPg0wx4Hop-f;ZQgsbHU@3ce@f7Tmnkq=#epOS&AVa~EAY>yq^cEcQ zlpA_HO$5YShtotcv>HA>ZHwyC`DV-A6iEb6fTXY7QwmRQK(-Gjmb+Hp6=ZQNd$~B#=4`e%1XqGT zQp@klDGaNPGfw16LJyiMYl6INzDn@om2H!=`kYP@7cuD@3P4`j>**#~bv{V_{N|9y z2FMKi3pIBvw_Xtz+ssAMyzLtj?CjD_3jbaH6*CgSUz@NYx4XMFhThyLCp14ET4}cAe8fnfJN3D^7+~3D3Khd`CVR zyhv&H)ep}1oNUvCy;*$D&AHjAkXfBtR^R@vB~bVvDO{OSOw|+bukzz(#g|hThuRk3 za7QH%RmcVnUXC6jo(T@FFl`?&0b7J3NqsJJEgP26L$6-y5IsI=u~L+m4X|U-3Qr-% zwH3XvE?&J4tYQLrwLn@`zJfBOrYZ(JVKg9nY&^{WaONuqFUY02yqkbn|yrG;B%Xh=mSsNJSie@8?YqcP#C z^7&ONt7U@bG0M}A1QTbk>TY^l)5_N$F>L&_0+GnWP{=CZ3epQndDFy9t_X5=ZzbJU zm8X(EHpl$lQ&{D&N)QR+xBWMeUY2_gNI2BL_oQ%=#A9?X`ynTjiAvt9)uyUm{mPM% z_k{1e)gnqztBm>Nyr+7?a6O=M()KgS(C9kQOWXEW`wl^uv;#5*EHqu zxS7&7A%{P>+a07 zAR5DmQl1@F_UI+Eb?c~Kv4z*&ixb+|OkS5Cn-g{)7#ZPSkDnG&-(Kgs9wPaaek@=A zKU&hi&S9Lsodx(2t^A1KegvdHg7hDW%a3I4M{fH6p?ux$7&O~wog8qt%jOc>PF!cC zPI$tFrxq+c6U4gKIJ9JN?L5D%PH^nM(|OcXR4zwP59R0OVf*)vtZ#vH11&a@Wpu~x zFS*UxGg!_l*iLLc!vjV57_A)M{M?|}pz`j%>s+1>aayW>r`3)J8qK`6vVZfhrV35+ z7QVUdwvF!{dMs=^l?9-?aXr~5T0OnX z!ZWZDRYm>dsD;9b_6vu%D`c;4!|SNM;=VDYaec1)R>`}NjzV2F5L>-I?H}@cg_xI< z4A*uFYOf99aHUmTJF&cVA)HfwS=h-yA?UmGC1G~IZ4>){pv*RN7#i|wYhf_c#DmD4 zDZ4vKiMFS?KPyIoN5yB=kF@^ zxP!A_8TAg9vW!(tvWVGXxM`XQIdjsXy)w5N*=X)|c1Q8Z$k3%FPZ`O#wx2(lZ0_>G z7hdP(V2v?WRylBcdRJk#+Vbrx>W|=MsR^HVCYH9<#*P0PKT^Sth&jGnvbh3Rs=lUI z<*_K8@MtGjfg4JGo#oAjZM?C=@_G^2 z(4TtzoPH;z4{b|^F9g|>lNI1P%##qd16E#*;F>iP&7txv;XCSqf%p-TK0~Di6*kMeNJzJ_KF8Qq4Mo@jwXW$!cc-|} zi&wp?yS@mRs9}9q;ti8~+3tCQBR?nZs>fld9(tCGV*2zJ{01o5uD(XYE2)bAlFtiG zLSG(z^^-_TuKG8taQ3S|E6j|;LS5|Izk|V@{O^mebzW+OQS;-AaGTRR)h14GqK-WH zpj7wft^umR?gY4mct$WBAk2OK?rAj%PbBFZeL3nW6U}`=4|Y-W zLy3rygn4%nbHA_3h4SAlR2!7oR?MsJWb>tXH2&3_{LQFi+ZM4Qh#b9xvq6$dm-}yD z-({O5OlWIWxWqj6^^g~tJLB(NWio~jtWIbf*DQ9_GyVO}xLyy%@R5C-teyH9ejV+Q zml}3Qp?Lb(d7}^c>)2$x2|S{u#0YiP(6*#sL-{DgE?uK-k<8&Va2+#QO~EH-KEWI^ zTw7SLxS4Io(2equg>5ts*!WPgY3V$_2zc>KS+bE2EjOzzKk>}jp#sja_;!_ zNTqA$a(edhEN^qcwIImr5HO@}{C>8m)3lJ0b(-b0ba^hJwNgX~f2U!~Tv@4+!&sZ* zXyWc$W1XII7=pW*j_k3#cBv>sjqqu5)g%P#ex0(FrpM-2)aiDkbC!Nfci%&!*SFw* zwknR)r=-f9f^lBUCD>Eu&>jz?i;eDx51vk#tG%VWAN0BnhgVJoRVapZJw+N6>pT!# zz;zCr{p95}{q+^1x{4HgN-NgHb^N)$*@G*DvT$BWmH8kZ9r=k+(J1-VhnT8k5+
Th$`;2nY9%w`S5;%{k?p)7Tk~Fo-^B7DJWsqcG*2L4AYLyKb|#{ z^`@|&m$5E`9!VosPw1MiCU(=`ga^=G;0?(Ir6`QT4O2zTOuOR0`Y zA(pEtkLC~)>VADM2WlI#>Yw)gUHbm4WSGv=Pg zdG6jmA9}fRue2@VH9Qiqf{Bh5i(hyo4Bk07qg5dhftS@)2~}cBdG3~d&R9DDIx}i+ zd1&-(qGNIIgfXgWZ6%Rx?X?Exi9nNTgTr5to#oR=J%AA;)o@;vnGb3uWhF~ah|)V7 zaH$l324a_E)5blTb>7p|bLQP@o8^tOGE>MY#{2O+RA94F9K#W&&zNU@Jdq~r6g!hO zE=QBK==;^vq@oYuyjb6wUwBQv`YG1QXvJPA8wYrqOJ6ayz151|9dZH%&i#>>3hx`D zXXm-(Nsmu)9bAw+_|PcJ6b`44u^HneH0_xi8_}OP|NbK|3WHcc*Ca1%Y6^6hj$r#r zN6?iUmLX02`gD8nAl zt3Y^n3Jyu^%g@NNW-)a+xexot!0O9cTkq>{u4oc6_otDF>=fth*|CL$9QXT^HJT{= zgQY_UN(TR7V($*-ln}Ew48vhL`oFyYtwS0HTL>U_#BBB_d9^y|uz2I;7AWR`;YFyF zh8UY>ug|^cbLowVl_(nyVk;(2Tw^e=^53YoTl(_Umh1i7H;!OUrQSO*fy+g#c*KLC z;h>*-qjk*r86Ra@9CWemPX8y-46)*tebQLP<6OphdTaNdaXge!k4#S`sU9*ivdY0%m(8&^iJ9ZPt}{IhP|IN)yyX&m2JbA^oFS39n&VHk@Jryni}Lc5(3jx&c1)KjmKaVi9B!iqVFUZgzY42+fe}2 zvG0p7=eq!Lc7vzjz!U2!7K!u)q#NZK8I0{2?%v+!6u^V763fn?_W&lwL$_%xh{P0k zo#fJ8Aac$cf}Z-nZV#s+k%d6I!`n>hjz=QB0kFO$3!rM%+pMuSf)|OKKB&1L>D7~u zR(y*BEde_%fJH1w-6vR(Oxn{^UwJ(XsbS{ru{gLrwaKgSqN9?_i%qv$k68S@J;wC{ z%;?TQz<61+m%;2la~#ktDY2c-HrvonXl*mOBt139?OBMikkHzVJs7LXG|T8;4zR{Axp#aQP7Nmd1cRI;=d@VO@PJpO|?{6TK#5EKLfNPy@)0S_k zCoa^YX2S}3_=;YYnA`^-2Rbjny53}=4%+U-eV_k2_iPO9 zpM_{|mJsSra!s0nu^@^#21Wm&r_Z%J_Fx!bmCGN6NH4UWzq9;K+-O-UL|gi+R$b_V zHZ5mFbe7Sdifbe=fO<-a0b)iFe|CzfChUZz>3YSDg!{Ze zD12Gv>ZsjcUwOt_zR;c(E=RrPQ4AvMy=Kz1hKr%@Y$G*aql6m4QRJSR{SO~x!Ma%d%ifPdy~Ak2P(%lhS8Xl@^+ z2`D`H2|PB{!+rQR=tToSV80LGE?`?kt+>X!Q51gaTNg+V4>n$K@ z-)8jR#mt^gVpUm|ne&Rax$@en&PpE4mW>k&gMeH?Bf+fScx=xU&>8u67n5~$QVgmb zMka$3|LkYnEw=4s?XbN68R%~^P_?|QEK_5#;xv)1;pt9U1a|P+8Y|rW@&Qcsc5|9# zcvksqcPmBMh<4MR9q_e!6j02-{OqJd5h;?eQ zW|o%<7MRPr)U05D_6AT*kJnj@(|z_Q3}A}~HCbJ3s?+vQjQ($IP4pBCuG)9rN~j!1O?hYcs-u!k_uG5J}xK(!;e> z$H9wrva@homR56Z`gb}}-&BDw>G7*KV8Izp1u+=~xpm;3p7m%hF57Ni-JG#C&nuf} zhT^Lq$Y7}rU#XbcSgIfw;ADB0F`Rz4sXk~czih3%1+7(mG3bpp4-klyJu5IN(ok~f z!3!E`-A2d*t*@Fn23_Ki%+A_qj`WqqZpn?g7a|IDSd+6cUHaMqAojgBh1VUOtX>!A z;V-Fr$`)3ot58f1y_pcYsJ1DVf%4$xZ*$J{^P;~&V?B>(eA1t%RN}x|FyAkBD#J8bM|$3eg5_oA>dtAL*I{4Pg`B*- zI;4ZkUS_m8GrJ|hrMU&{;G0!espzE{P-xuE24v}mcuaT~8f`_^mTO*!*!Bw@?O#vD zT_|p3i@3o6zRk2BPv>cKww57JSu!&NS&HQY7QqO$`7}}nF@EzEHWb4Jm*q@ic)PRw zhSag^yxye~>p)ap2i?OlielZ52F56T;G$Y30l{f$Be*?>qkwkj4sqBEa|7IxqLazd z{>ki6_ zy>_0_gXKpDJbRgS{3#Z1S>=jHE?|j?KLrMJ{n>e&oTB7%4mLv8@7&{$S{mk1ZK^K0 z%PFaxCiO;);*n2sjFP;ZTc;j}o_59{7i*A()+ zI@%ynaymqK0Dm#yeSzjeFLi1CIisPRwcr{AvJCVsS`qm8PaVKHXJySAvw%eEt*r*g z83@$s#3^<4P)Ri+bA^?Z#b=Q<{QL-g6F4GBYctxNrEmG{Mm|l?fdX~1mY(Z+j7s+r zH~2t1Rd2y{%U?0r`NPvfh}nawy&w_wZD=>ukCabkbCn*Xunl(dN$&^eKu8q!E=R80f$tCREZh5df zLLr*r1yC zEJ4*$T2^pQaMDn95z%1#++55ZAk_@kq8|4G|CpcA+^Y`{@LQT$iT3(fM;^Q9-=ZQJ zgvSn5oC~T_j8^&D_&6`f`HNGxIHaoLp(2F=>Rw?@=%A1Xa@9+4MYl` z(?B$!B)d{U-3kyxp9`|iq~t;^OIp$?a#yMrLGXZb4;ckbaQh{AxI}%q> zArmLL8dx@NAG-;HkOgy3S$gF!a%%_~Z*MDXOS+b7gQBSLka&>fcmpVff@3nj09=h| zMWKiUn1*Tu;<7ts5>@VjTWB|{Y}K@>^e`8w?a!b{93Z`KNpGe|CLwT?p|D?I53C)D zO6lV-#Zyj!I|TIov9S`%7WeJyjl|O}YqK;{s7ZOAIRcDn?x?x)kmTRv0}hOf{VsnF z5xY1IN|nRiZALFPK|BT4{s`QrhH98_Q2wY}309!i5Qur&yF2bP2#2K>?0BKk=0(m;pHY8c3x)gqXTu_7gu%Q<=KeKBW-a$Ilb*^&v?rN+=+ef4 z_0(j~fR(W}ng^8RNP)}7`u`fV*y`V~3qf0DDZhCqDtes&oPxFF>`Hb!nLRto-eiOF zzj-Z3?oYvzET)0n_Kc&!RQZ94>lOuEDIS_oAKVgb^RgTR6q0sp#p$wTjoYa*i2tSB0PwXeyCvt-I~Cy$5OKVI2Ij}E zw>D3@u~QX(%@4?613f48J3T{o!~T~U-%{RqLD1x%HS^c^{$*ieLLNOJ%Q&b9oQ%C5 zuKOGOhxahdOdpO3ty1Z{VJ)P5v#k7dbE*`Gg&x>^g zSr8FEGS?qW6H)Q#*z7FiVtkeX5oD9pgzR=L*V@tQSx8kelQ}==U3%Budw8iSt@Kp7 zM!H5r{0;b_NE^)E(+NttHX8nyqOS!s+Z-TpbMH}ypF4*P{n_N?6MmD15Wd0Rfwqdmt$G2GC6)hVS$0@PplY`f*xG}6qa(AK9$8=(Z2LQ zMit5x(nQqQ<{B|5B_&UP(8Xzvb~QnY=NMF1K%fP!R6*wIR#OdSH#l&aOM8$)F1mR3 z^aYrZy!1-^7W&Xtco6cT%w4CaAy~6A!3CSA9I}b|$R2l_iR4j*r{dcrK#lyLhLq`w zJ1x+zPB#Jk%KlB`|A5KdfpU1uSGRxZVD~m~WdR*GmJka@KEyczc(k_zoSuMoBWcSv z=bq1>%f7*T=*{kJQ3Jrdl9rxra6Mss5?ptOG`&@6B={-ujafFxShjI%Hh)f1j%DjT z+{T@@$Dn2ni&z=EqBLMX?$OhHf=lu3wQ!8KCb?)f+ex@7_3aHX3&gLMgjm0d$s~w+@ zG+2`C_lk0gLhfyY44mT6qf@8zWE%;bTJR&Re#2r`PeTOT0qtzUOO0UnGQRd(ZwDSz z4)0QWS9SE?5`Yz_16d#(wAj6Su7qM9Z~x%xHm_C?I=C#Aqo#U=MRBoMpwf-AZ8y1r zpdVhGl5ZqT3*+CBMeoHT&#<)RJ>T&$OQYT45kx^^LL>(mabb7F7MKw!#9w&&g(z%4 zO9{TsUQDl!c567bGgzwN zGS{-RCQ-9|^bqtH7Te?9Tvcx~E1vgT0?L-iQjG-JUw8Y8v-;i!ec8Bo`^t8E3Xau$ z>F`Y6rTy<0{4X{$?>GU92i$jwxt-g7f~(aZcSWf9$!@nOOQ0(uKvyssL;H8TDaMvv z5n8BvpvX6t%C8#_H70-OOfUSMfB*~Y*f>L3T4q#FqW zJ0w632P*B#BL6QIfb~tm2TwV68}AfuM1~-Gti_?rCw7?ho4!5rdvozuBfhaKyO=6o z6HahdAnsr1@ZM$Luvnq@(-4H;*C?6qxclol|LsQ?=f!`ud>A+ZHnqwGJ^UMmcohHj zymK)!WHJ|9d_`z?rhMVku7rygG%KMJ+n5BwPF%b2xxNhXmBG3~b z1HDSSKg~ieQTuki+5 zvMhY<4(GCS!!HDLVvl`4Q5X~x!F9^f{77Jiqw@$kT2^uw+Fwz4<0sZ3XTHqbKqZra z+1TRcK(Q#lPfKyum)*8cKiB921CNa6_aD(AA8)-F4lZK;7M;tahYS_01u7}so4&o^ z(rXZ<$GWn4`u!c~^5UEE*%YZ*>m#f?9Ut2qpN5$0Xd&dH0=F-Dd~}^hBrVy7sE`gB z1E=!Fy20h<_Fa$yS+k7+zvJ^GRN^0q;{FL*5@r3XeIFU@UFB%`%n^jIP(H7??Q=5~ij33_s`@qv3H1GRErI1vw*orfg$N+ z0f~(O``IYN)iS$uR*3@gu}G7kXw*+r^*nfbYrL_Gb12_$tspQs4k{#Zc>|6OvEAHoYRr}U#mRTl6RR^rpmf$>=ugy+i z0yiss{x#ruPL-R>!s9MZn5m)weJMRhc{Ya}Fe*a0nBsI?Lh1g;hfm2u{##5#xygOe zjC-uS*TAMcom6xR(dl)|URThRM!0hHEqcZl6XRKpHnm3y=m;VNt}IvS4|&k^$o*)( zqoS?ocA*0S+v#I$Z%vxzM~mq0gG8-(W?ELeasNtR&FCzaF~@^SGF`1wHR@+he3%T! z+yar#y$nUz*9IPr=-jI6Nc!Ee99_qGLX%m5x3K{ItQz5u-7YPT^XlQQlQ@sMsXkIG zywcNh;ob+bkx*w}Shy>eUZ|kU+z8pugfF!VLa}C-9<2HrDSEigXH*$0qAV9AjTD_W zW{E?62c9%<&WC}d_*c+Rzo@u-#Nt36KWB~9j6%Aq+Sd5;EdeW68v#RyJKu!eldo@Y zZXU1bw;CCo8l!Q3WIm_r!N#oa{rPyS?_5@;yV%>d>T;nCNBU0Lw+j1FPqrY_pGEvr@9!&h7xwG8#ZicW$@M2ryDB{ zcUficD9YcOghj3M6eW9Zwx!@eBSpJ)?GU-S!dBtt+^U7MIaO1>fnml&eMYodQd&DZ zkC?;O<@xg*%}+5-J=^ut6To_VR-e=)qnXya@_viXU+~QmiuYYAFaY9GSQX_*96 zj?agQe|o$=TRa$)j#eZcCoG*LnmFy+<~lMIib`ZIzv2DNd|P`JUiXTlImguV(?H1K z68ybE`uQ%bv4fNv!Dd^+DzKB)9y8BqWZ#}_2_wFIOE_#KHd=sIR7F9zDB{bgkF#~r zOOJe&$DhU}`?l{lKC*2`9uJW{>pwE59~r}s$mmC8^gkRK{is=PZ*%=$s8RKOgkd3& zBL=Mh2>4M6-rn8&Kj*Iq?;BeE7zx;U+6hSfudK}vS=s|42ju*@HML+V6+x|0Lrt41vU`!;ci-hO7Cl#*Q<6nO{A64nQ1uSwJ8sc);*u zP=2|=GKY`L;3&31T5A3vc;#e9JlNx6hcvS4_mJOCgRP)FkV~u@E#xU{{|xewwJ*j7 zx%&;uFW?202|tGX-{IK$m>6RsW=OG~0bYe0Z)7esI|MjyHBI3R@n zm8;L%!q*{wek_0>?Ln!Ie!Q@ttYl4@2ICf-z#}Gjl(UFsx1Fm2!_q_!LEAOpH~8D6 z9MuTmtj{Z9bG#sZV{v&K<;IuA2!9;FRME=W+7s+A{L3JtX)z+XlX5HnXn?(Q{ZO&8 zte>sFT*_a_9*HgT;_?*jM9+S#KuQ65-T#OirIjGPuDd_>8C~jBX>!6 z2gg@-MuJ8FW+oJ~^u2LsUZX2jTI`!Ddy8wiFt&5+EWS7ir4;bfRQ8wm%PUzE$t>Ei z{7D7m2p6=y0?_&#;J5yFRX+xI1R953o?}~ASk-OC&r7_0#gQoC&5*{HcR!N!>(=ug zG;{1|s7E{hnw`Sa-J@1kEWVQaYSz@6XWW&e*rV&OQ_6#FT{_#v4Ll>ntQu!OP0(vK zr*h2ZG=)5>%-P^@{QA~I$Hbb)<~h!|O7vFtY%S zyNk1Tcx>W**Z3He0SLJPuS~~PA>0})M$FmKn;fW~EZnLEfs~VjV7AnBc$!_JhBK#M z+2`3g;>|e2kLEmJQ8vHH=-(rE!EgkY$d{;+?`bcIcyJBA>ZxgbzRh`Xn2Uk36+`j; zx~5=U7^~uUcafPS*`nhlpN311%Vk`**A-@7EI4JXB9>e5aeZcWPmsNXl^^mc1T82j z5Qz~YBkfOptEcGj5@Nf*hFBZqAr3wG1h+~G5?!a&(O0{T?YqSdY13V1O6#gt z;l+MnLNN{m5y!#jO|fP5r|S*H1`ju-r8X$`!NM@cUAXn;;4)AfaH8N=7g(1W0zR_J zG2Xfyg#E={+uX=AQoOfu2@nv^Xh6YXo0aJ~&yZ9jdtb@wjrEMKp#gvKDq2NXm!v;= z^=+3KvArh$u{cXReywmK789dmCEX&mZ{xB~Xy%`;bV`0~`=p*BK873*0fa~L{#VPlE5V(hB{ z2%bNXlT%?eowi%NVVB&f2;dBVwL|yZjQjk{aY81TbPE!`>aA%k+9rj}iIZr7UY>io zU_agK&N0blqHA=?V6cKFe<8MagvIdu?xOXY$_^P>OO5o@)@?4e$I@@Ks^=)DXCe1*jR}rboaILV)R9N^mnpxFeuI4Tq`eE zr3h!saNhXZjiB|44aqBR>=bldgpYkZ)*5E-@F4kXpJMoj+=6|DACP>bf{u}Dsy-DH z!+_d$#6ox93aV%1wuCTUZVFZJChr#00>)SkzN^M%>w8DpJN)FZG#VDDi7oTbD`v5( zcs=by*@}^38qLVZZ73xtIls@df>rmiR)AeL4q!=l7FB+5WDGV-{Klfi=~JTjt21%d z6?w1Z61V&og_CZhEBo4qK>erQMc%E1o>W?Yx=X7(f-(AU!<^OaeZ{x3mS!>dhXdZ? z{u@j7pd9n86DwDGCTd_ylh9?r!WT2M>7L{7+i6=?zbOZU`U6$I3U}zq#^p(V798sH z9_;=aqb-IjCZ7^a@w#r`;6O>Y%)WpCT)nndk|S8XS!iO-;E%+p04w?xHWF>D=o_&4 zqN@*QTo4_D~;(BbDu z`4XJy1jnd~|CqgM#oJfByd}lmad!+TAG^B5D_{Anm3OHW)bbop+yNk1TpJAsQ*ZtY zIkgm3duW(+%7ET7-|VSQHi3&h4O)DcH3!c8P~k5&|u%5lBn5ygOA%zD#1yR zj#zzX7i{nF^+MRLAHbOR7Ec-ymOAmyH|;f>HG25hdncpsm}%v0S$#x7n+@fyGof2= z6fHpw)J%a&dF$_0*`V=?rDpr&=zSZr8oYp{&K;6$(k$E0v!lrr#Ci~cgNuXFKld)? zSO6Mlq=j`7SY2v2m$zEM8y>KA7`+%Mjax`)NM4K}l~Sv1ybu4)@}ndPSW?s}wp0MJ zrNoB0&xF_~k|-vdsfq@s#{(@OIfAS4%a1p7XO1yLvvI4_#*PV|&^U>df_~yVr-0_o zPIOa}%WD?DYm*UF$?3m-$t?(5iM+==*1~Rr@+`vfKo6NKt~D|!%5!?W;`ELJ|055udulw77I;4BK z{4yaS0Tt=0%E6W5IaAytl0O*A{_-RYY@|Ytwn2VP+w(e<9poOrfj{&Ck>k0vJy+G( zZi3&S%M&C>?I3@uu{{Mt-mRLUKV)|^zWRGswGvXOV&;Z^pV(||gjBprR~OC8x(9_; z|G6#aU`9<)6>`rT3|OFmxBO3Gss)!`rLUKgbGf|c3RPsvV-klp#xw^;QxN#c`~0z* zs%5G#Z2=V#8Uz`RIe4K3adrZ`9GZ|oYsldJoY%vc2{qp8rVotb72lsk)%z( z=Bn$xYO(n^*=AjN@}1UD0nPbCLPPVM+b{=yvA2Y)`?H^gi)vnC!p3ub|K)Jr>h(Cx zdCnK6F3q}7Ytk$8QW8fTqnFJ_$Yc0-_}kK*xn9s9Ob5O(Rk>I~H6C?R*sarzQp5~#L zqn>+LZ@?kXKOm`tH65u3ny~PA-=VoNYYb&Y|Fby4cSUR;Dml@Y_<4@&T8ksn2D=eQ zTz$FGO>~Q{=}?@EL!+q01e1TKZKWR_P#1D67Vn==FR>8|Xul%F>zv+hY?L z#IGBj!O$wyUAAc0XY3^}dhZAG{2qtsqiL3Rcl5^aFqG)O@(8~^>??V&6n+?rHTSAn zr2LA{m!ia^ zhGDWbWGg8aTfv9Axk@%#MVojzJdIp-i3`OXV6i0=*gz8FMkC(Q!~RQOUA`Lv z`K5Gg&r>iJK&|=l&&&_QOVxo5g_^OficbY5%ZYw(;c{fEnQ+m;9X?k`-6R=kOCta-xYvNE%La$- zM4IJ+T~_$P9bj8?HO~)(w;mt*DlD%!s0>?JNe@0M#%UvrNb&+9Za|6n1@tG-ao5NV zsa{r8TC{DmWYuytLHCeHa`LngR9Z6GyqaVgSQZD(%spOTlQ?`JdO5Nfx3_+dfKPhd)OZ;E6{1fnK-sZ>lYoz_+EO!67#D4bA8I(X! zZ_8)=vz literal 0 HcmV?d00001 diff --git a/figures/example/dataset-otu_large/kfold-5/method-rf/hp_performance.png b/figures/example/dataset-otu_large/kfold-5/method-rf/hp_performance.png new file mode 100644 index 0000000000000000000000000000000000000000..3f51afc8567dfe500e6a48b9b1223cf73d9ce7dd GIT binary patch literal 29775 zcmeIbXIPV2*DoH28PJh{iV7md&WuQtC?F*|qSz2;9O*?wA<~NwAcUX@h&0OxN>xx$ z>C#&QL_np)&>=t!NKXL*2_fz5fHTi?&WC?F=Y8MnI@jw5FGKEK)?R!4R=d~UiN18v z1p2f5&kzU%diIR*9}viTnCRbnG4KuQzA+a95rbU1@aHM;=el+4e)!=BF)^_pfBf;M zpMDY-7nhKbSigS#h7B8j{`u#P8#hWyNcgc?AReMFTZo=P6Y)8MMXs=B_(BLG&3{1bm`LXzyJQnAAelFeEH8m|GaYL zin+PDg@uKsrKOdXm9@3Cjg5`1t*xD%oxQ!igM-7>t5+Qz9j{%x=H%pb{rdGAH*PpP zJG;2JAP|U~H*dPSy1Kc!xx2f2czAevdU|2@f7Tc=+()qeqV(KYskzUw=J$@+2`aF)1nO>C>mr zo;^!WPEJWlNli^nOG`^nPtVB6c>etPix)34Gc&WYvR=M?`RdiH?Ck8EoSfIMU%z?t z=Iz_Jxw*M{d3pKy`2__9g@uJhMMcHM#qZv|D=8@{EiEl8D=RNAuc)Y~tgNi6s;aK8 zuBoZ1t*x!AtHWZkI2^9NzP_QMp|PFMq5?d$97@9!TN7#JKJB#}r%Lqo&E z!y_XjWHR~7moF3wWps3OY;5f7*RSK_;}a7TlarHEQ&ZE^(=#(Ov$L~QDwRf~(dl#s zgE2QZH$OkWu&}^nG8Y#YmzI`TEEb#1US3}2a5yU~D_ky@$K&z&e1Sk96beU+?9c!M z8@$h0`9UD+dqw}&1-^m@Kp?vyXN^z%8JJ2P4hnlIVK*uKbU}T~`Ujgl#Wu|R7ItHp zm_cnG_V2SAzA^dnRPx=Yl#4N8(P90Mw(WV|>3il&XG8uBLU67*YxL{bw zw22LKDD*ce7j>g_R$)qFkH6wtu>6tEjX3A z4kGT{o#}YFZx2|on6@j?JNTcEb1`tF9B)6 z@B2R*<9~bd|8E^*jbR}kZ)p>&+h6hUUL?D8P;jIFGLRe7|7cPFkA%&Cld&}4c`8rU zMT;h?h1&Aw{W(Ux0X6UW9E0BOX%uZ`^IY&uozdWNIH^!R<{t zk|;>nMG>d&%~f3XN?nO@lwMbKM#QDf>4O48-kX2@Z7jCw^y-7}9{_j0GvPZEzQf@= z3jX_`z+6q<(>lWAT|l(>u2{q`dDgI@qMu3EX8T^G9FSS1U&`R}Wydj6$XIou~ z;Iy*PO}?yNU4o{}65Nt=$mO?|f@l#LZkW0;zRJ<|Q-0??5p29hXgS(( zvgM*%GWl1XgYx%w$%kp~Rf6`Vcfyp@aLeHV&|1vL83L?*+M?Yd#HGu?g?>bm;c3y_ zHz8{`2y2I5vDpSS-UL1VbaO)F#*~Sn;UsuBj9~q4B{iFq8xaE!A-G_**-O;yAQHXj zu_-gsHFu5g+M~p|#%#_Uy;Uc@8%Cb8=B8!qG2`hwZOO3qblD4`*t$?u@s)4Op&=?c z@3X6}5s;Hd+a;>aak!iIxUbtCJWcTQir?EtMml0Wnw(FqIt0$8SufHcIfv5SDRHJu z3A>khlHvMnu%Db6GeS$# zPgQN&^$#_Hn6BFE^EELk&b5BDj>jnzpAr((Qa?`b?<1(y6riWi*Ay1j zp(83}Q@*i>7%yUvjx3HF*kzFKN6V)jU!Kk8jO)}fp@dZZmacZcjTt>KMn5{jOvg?F zHsaTNN5yc$;*n-Bh-0rMEOn%cRpV!}KLB&7Cox-ZdWO~KmeC0#@FuJFJA~T% znRoFhqf4j6tbiQ>%yS%|MCTw2()3mE=@zF)I^!dx+wGR$N7I@#LdIqZHiFns}{DfO$V1fJ1m1aQv9WDEPgXo z5(Es5u>r7wmrbHiks4d9Uc4Z*xXgfE+AstonzQ@i@!!vo3#+lGb zo=8X>3O?V(voHHw1VIAic$Wc^9cAV|z5J9K1UCqx8+0yV~xV;Nx-_K?98y=|iv#fC;E6v({!p16zKxqq>g2L`a#iiiznM*0b64 z4~7QVN88Ub(qe8C;Q2T0bu%D1%*-3yVH|6)D|w=?z}2CWV`|#P8z0J?CB@huLv9_5 z?5pq!w)!{V_jBF`KvR82ywZjR3#D!oNp;TiA(h8Le{>FZm611eEE{O7{X?x6eqdzjW<()OqgN=hS_5wagxXV zwH1Ca!BeS8N^v@GvPZNzN6qf3cjP@I|{y|;5!Pwqu@IV{_9b2 zqVIG2-?aeWCGfih{?|+3-%@v-(}*6FKHtt3M@D6?3*IjuwhzOp>{XZW#9q6E{8j$B zMdrE>KBO*dl){ROSoLKhqz)v5WA%@gA5$p0p~*hD+Txy%XKw56B&o0Vr^FV|!63h? zJCGHj`7+|Z6>EcTHFHy2`gmd}9}#8`a?9`2Ow?l{{Um&FYmax=j60~~zGkY)reb>E zIU@{xHnBBUN0uuce&XFRVXC4aeDF%+`qZtqUCo=JpH+#FymI;(-CGf7HbGH)S96Ql z*@C;_>=#=SmyzL5&0)t2jm5kZhJ$;O1cJg5ezry3lvNIoiVJ$b8JR2H3;*)Fgu2ag zG)y3gt_bUumq#L!P{Q(!)&J z(pU9N$H_;G!b4Jdc}AcR*|GX5`8Y8tE_nG~pzBIcm^z_W@UF*3Al1~>Xsic^5~b&M~jk0XF7#qQmyF=3r)uONY*YDnkAOkQDwM^;L!dMR1S#dy->5q zG1zQ>-t5W!`;@%QEr_>qaDpD zwU)Md<>R`V&SpIklgd4`UtY?sEHucgLa(N$n*GlJ7M@ku+*NkQI|?rl|Hyp7*FkrT zcxsQhDpsBw>t20%b*L*xOls$`gN|_cmm;q}GATvovu$<56EDqYLvvvlO;;D*c}}-d zx@L=~A}ay*3ACZ)_YwdpKs1~zZl)%S7JU`hZMR{TdufA|?xNW_81F)^|9kt9DqlM; zazfVp8)vTKJ}Yiu<4_O1g$1Iu_q;U5Gf93S@9fK7)y*{J9=&u*G+)tCy)T-LU>*X= zFV(uBibzY_@y|vUSb5v|=*R(=Qi-7DibM?@f<9(~K_4-(z9^bX^WX}mp0eU7JkxzGlaevn zF|<@D2$~*6l5^{pM=bLH7Ncwa7LB(7G8^b-6ZHFFyrd5P7DZN+9Zr|L(qxCT_Of3p zu3jbJ+M&~p{kHoRK1Yd`xNqDTuLv)ht(ZBX!R1D1qaC=nwI`TMr6(}Es#X~nSdHAT z;G3}5adw=Kxo~5>Z>A%QD-58>(jK%3q9$Wp)hso60B_` z&~?dXSv6c(>MKHi{lP)S4KmCpU&6!;&)$z57=Y_LPZxg0ak|6AXZMMEPUQ%cyq~7$ z-aVmw%z618n5P^m?2Ff#9TLVSK%L+2U*#|Rj8Ixvl4q&q)0p{!)@)Wg-(&{ox6LyJ zeXsC?efvct^{X%+YL?Y(>0NrUMI3YMsdo@@)A7}6q#ZWeEkk9EkB zX1xH*12#j!dzbk+SZe7|wv5sdAz|9cLHqd{o=C?ZGaXydHu)KhzxyB^{xLCt!B_VI0l-yQCs4V&R9BbG zeJ+$xC{Si4{CNBHZut(+viPaLAE{0k+!b3zcfFlF(D6&=ZfLTERkUcCQ>7|fnvqw) zUTR8aZGsZK%A{u3`t=jCUaZ;)-7V$z0*#Z)%lei!VCeKtSHn_9c%CtAlOrZZTtZ>j z$S=nDz1I1{&+HTCK+21HvmAGCIUsvbw6)?P%|ZQ)*yFH}7lxx7SKuFi2Hfwpg)Oqf z@Qb$lb-zHwj{HCbO7^`p3-f3jkh$1U*rG5D?|~OQyedpohRT+!gw$VE{uT5)tsx(9 zs&O&>oMoQ~pX(mIH^D1+q`exyk*oCynBhHOo<(LD{%eS6;D&cul%0&g#-Z>}cu(6` zH(9^IHtp7uuU4fzH=>HVpFVRAX7Iv$2tBEMO=@KkB9RI_jt85dv@`vKzXM%ZFHqYr zZys2$cfF|a4lwn1<<7xwiL8hcu(nznc#JNh0!k0j%vDmr-)GUxtEt6V0 zDK`Wzr3*3Km=Ng?XXc#LHLOKkoucz-d$$WVD=dtrtW=NN6foZ21rA4LR-xDrRX}C- zMDz)XWP6=@;`V6qNE&yie)ss+gdVfz@>8MLI$Wr#c*|$PJ zQz;ydy*qYHgzV=MS(~Byf@*?O-UXPZZJ{-5tFXJ{obIxx2;}vKu%C%@Gz$goe#Nb# zQKoCI{B%AKQ=cH{#-KlGm|nPOidpO|v~#ie!JE)zf@ev4ChHs5$d^!`Ag)%@k?g}S zP!5foM)D-{{3*n_IC97o4^Nr7P$HXo`$ctr%NgC#rz$+^M?wfuRX8`SEZ}O`Plt&W z%{G#q4R$|A65vTDq21i5Ukb$+*(w+QV{Z8M^+q?U*{tI{O(z`e~ zY1FZ>fzOEL&X{=Xz}E&vLF7%WhWp~;wB+e_lY+&C2)H6E3{nzJ^lr@%L>VUhMW_yj z>{U~dmwQRGA&|X~yX;ob!8)GRHt4X~9roi*qW3@{HMC-2NLV94?9x`-D=+snH%AAV zMw~B-leN{-=f-8#Bvb5@MwTKW<_>?th8A1QYy%(IvK`2NDTu3aHrNNC=W1t0Bl zR$)az>DW-T>lb+ec~Yr(PLA`zBE*mWHuLro>RUA@Hcc`TuVacXaaytv(9(~UNx+3q<&dc%lE?4=h0qP5 zG%!`jJiejDHfY`{hYWfBOl>)O-OU-&(h)PaLVk3Eltj^^zYzUPQ$f`C36&4E#Fw9t(HmM@F9ySPZlQ(I;p&FyQ9Eb=oMM zD;)wHQM)TJ&CznZqWK3M*8;!vC(g1d6W}HBV;PZdhqL|)msPUOSZ3`s7TZh9j%iMq z9%=Fx^rmskH;6{NA6cHOqFgq&3Dw_)ii>O|FYvU@(5zMsY?*D1-|TFQ@FYK9ROc$NJH-e})Fb&`8FbfPqzWzdb~ z6zBxv8Ampzql1c9R+4D;{C7KLxAV#EHL_{Ty(_jTFvyqYm}cdX9g-61npK50a@)cC z;tRI0QZyTxU>8_v$KJ?z!&avHa0|;acmxB+6%7O?Xdx*nA4{RK7UnI|k>=di^Am8p zv8Wf#H=Z>EQmI(sI|WINJ$0H+m~DVbIK`{IwZdy(I{jhhU%8eQ`E z6XuPy)iD%&?zpOZYwT-p4YmgufK3Oq+);n#rF_oq*C9hdj2 zxHDQ$F9&`3Ey^Z!__qVc{- z`l46zvxpc1A{qCFZU*y~#n#-QhNd-mFE?+*4hT`30YRYH-Z1er9uU7jhHO{+wSAMm znP;*p+^F39Ku*%3)J5SVnG_lF!V^-t1C!{Y)6Te+ervMu!1mOjA+5~YbyYy^?5~mQ zc=m>{CBO`%3fjO~jtEm`N(>oYdo#Vaiq#u%kmujUTSytGX)Wk3vhOHI4$o{L)1(XkQnvfyXq{$RVsu^~; z>`zGvFA{?9uk^T$-26S4X^Af>R$IqnpGI?WBIZy{a-f9-m{EA`gWP=p6q%8tT1 z?-dUXM6wKdASm!RSP_zx-AwP=A!!n4N~4JpR!*;#k0dIl2L?|L&ZfkJ3YdkzvN@Ny zvscVKl65*T&zoM+Vi?p)0jk4#?qv(_HR37}sLd9z>($WQ+8D>%-zr?5-wgGbuStNl z`|0;})MZl)QCL(H@9fS&2m2}%yKypk!dL})?FK3PsN$fNq_kX0weNAH+FG$s2KAWV zV#d+-pfoq6s1|F0Ijo;NaigT9FRdRIqZ}FJ*;SicxszC5o8odnX?Ta^tj-17cTV0& zjkW#`Zpn{j$&mT#>>#(k8QELudlu|!;;)2cqhp%-w0MqPJaZqN%-aKYHF8qZha$6D z%0s-0WJEAZUtKo1Z)LQ7|x9(gg2)fPt~`WeG8zYeWo zAO_@WW`S4lx6RY!c4h~W>8*ZViIvrEw5GJ2XhEEv^0E2pY(1a({qCS}hZ$3|8Op8j zkaahxERjxVEq%(V^cj zVm>V9 z3#?r=pm1bR{T}8J{WOf4qn8e@|jDH$WUuNy)|@ zp*{m)$+z~;Z`8=eXFRP!_n#aFCr?&OXNr2=Q&v5lc!iV|79dBo>U)uQ5SHgXpRFp2 z*`pWQhJ5;T&i^xStjA?1HXWd#IeSGYTjMH{sM1VI?Bk0wT?(pOpl*%r+A;}N)iIO~ zN9i(=MF5W-j~nBsNJq=2f&ZZ!?EY5p{NZ(=z+~j06?}X~Eo)rp7u}PQ7mkqg&>^HU z?{!?Y#5s;WQp79G&_(rExO@C#c^0{GQ%_ho~JD*W#H)h5~+1aL`V@r_l;pkG+FM0D5A}} z-qH))8~^QpOC5jr+Ef>~Cw!M(W@Vsz#|Ee8qriT=*r`$>g~}F)Yzi18uP0f)O%%ZH zb@Y_Q-v&hjJ)%6$IGi`?)S zA>oysxF$%EY#Qr*77a~n_866IHfD~f2 zsn5hgjvnmc3w3c~k+AB9^XPJiLzw^01I~1y3c~*eS)@CuJNu-*kfp47)PKClMP3}> zU_(q~mNLgFK~GQ|u{yRSv0Z!8;GMQyuh-Mc>n%l5@>|xLNoG1)0w20~_LQ2m0?C&T z1^_1$SAnD{snee7cLAW6Ss|JdFA_+QRnWoCT)~i6vjgIuq@RG?9g39jNEZk|Un9_W zM+BmBEUfn|TKW3^OvmS*wFn*s_#a<3JmzxMBYy(>qzxj_H1k;y&Wc)i|E-CALslg4 z>AyybRPk|`EsL4o+rZDb{!^x-WS9{+e6uR`D>6N2Z$1N2wwt8yM^ znPc8bWjZd=`j1u3*8@O$hO9*4l@7*%*6&Z@k(h!jy8f#;Gao1eF5)zAwyy7Vjod*% z&g1RSJ|&wY_=gViAG~X#Bd?UM3ST1iTBT3_$q4W&1#G;oe~^~KLZvKQxFO_Jrem3{ zVI4=1hN%EbPr+R}e)}3bH98C*yN8f+jD!*9vkqv=J6sTc%w|?aLZAma9dRcI3nIl< zrJ7U}^+~jc0K@jERCqe>+U$lJES8PUm%-KyURK*Jzd~Qtl0dI|)qc6}Ncc-s%nL9L zudvQhv-R06?~Dt$HM`7M&5TB3MX}(jQzp`eaiaQhAO%rn)(nTrQ99x&KSj*i%&O;E z;$|qH8TSV0Q`)tD*iybD0d{x|8d6?m+wJ8lK0Zj?(%?ha>mBfHF6!@6BGDjGy~R3os_U;Dd}k%nEbziEZco|RGF zEBto2*XYjh(63ve?9XcF1GMr;np%D$K|HYr+yvJ$fv=!b%w`seQ)YDd=kK-fe+?4y zhQ(mmk+i0{u;MrCGF+V6*ECjtx2vIxpcCg=W>i-+S8hIAe?`$1WG0cr2HNve|5CNC zw}7F#U z_jeJrTU6AsRY>R4_3jO@Kp*G7u@di5722@vg#dAP)*8uK_%=pP^QSt>?jk=>3LvMohdx$n(9!eH| zUDOX1a&z@T(l7GHM1YpUXJFIAt~eLVd>CxV9LJjB;+(O$%5^L78AYJXGr?!V;jFvlC!0RojnbM~)5sW_zjY z@(U2l!>owO0)efaG5$_&dF*h7$Rfa&+8qg{aFP6$B9?o3H5__}Nl)0{wE*15HCc{* zp;1Wbt3c(vpmm%-kVv~$2~bCK5lMoY{De`_%smI;P{+V9v50>Jh=UVdt{xF%jPJ{7 zj-LY2saLYdH5rs%6V8xgK8BRB+*_bZ58eHhs$8lO>RmQD^A2Nz2XV#DW8p9SdGFh< zX2pq=CoqFJlB;6SIBy%<7*91TvmVRU@7cojfdRj##>_=i;mFY*A$)zXh}Iluhg z*6jtAa!@aiGJ#0w-|%7bz!M6*;G($-!pWkj!NN5{6iBY<0LuLs;#`&a?9^&oE%|a& zk7K1Em7oN|lYVU)drjKuU9M;^QB&_Lx&9yCN|kjBwnP6HKj&wV-2qA{+$P^%X> zt77%rlWJuSr|`hpQR)Oi&zjM>mcFK0PFm+j$ZSO#<6Ug>m92VSBo2CIQvQ{^aSYk6 zcVtboWgyB%%DkJJ5P?0>8c$$yB7N%*K=uOIIHN{w<3e?>UW6fLqnK>x9- zWPVCDWQ*S)CeA&>kg$u(4cJ5s)CBJjEDOlqoz%JLskm?kBurPEQ==x*KS~LkDlnz| z@$igb!uv6#VHx`Rt`b`D=`6=pcPn5!-e;^!N76@$E}G=!DSUxT@2gOkO^^L80gu}k zm&5gi7Z_ze3o-cQ2qn>W*}~ee6JPd!^qzAQ*R)6~+YTjFTud+W{HPweGfHbcR}{!D z09C*gW-&K-^3Nva3&8TO~RinKpm4B{6(?F#9Mf@x*2b(^6P;Gj^US)3wqha`IhSfF)7YG~03m&_NMjYRL+ zw)c?*>~sYLBv{@ip-u;@qv?VT`w5vmx*|OFIv!@1RbiimlgVend<79w{7)$GBGjuj=ZpOh=LL zbu5N5RjfvyWf*E2uL+&7&aeX~V9ttbx7x>@h7G~|F81ZZd%biqvmFADbjmkx_e`!o z&ounm^`Ah7hos)$ztmtaJ!rt^8+jzltqnv|X?)Wl`R>u&s>~`E$2elX`TdC8CF$@T z*Y|%UdUL)uhXWcKQgSe#6DaGy!QLsqcTz(8!?{1e6&(Mj^otVLmTFMfQ;pCJ&)cUZ zAK-2WeB}Q5X=Rq*u?3N>v4n+W_6!;TSAB~twC`d9{3Xa6m-+a*Wkl_L@J*&I-G)sh zfS;V1>B#N{!8!^X#65m`pe7Yj=^V^_2+V%S8;0LQkz?Bl zB(oz)u`_sCk5T=V=`eK|Sji9fBL6L9q;W6(C}LG5qF;;uIKL6>>)u`RYxyj=&~c|m zWhO&x^TpM{C^k0^QyqY|{o4u3XLLRI<}j6y%{upEGAXOMbzXxsQ1JO!vqr?4ClScx zX^#`(g2If?9-3Wha^|q1Dqz^>Wwa~UXvxzeZxleDP}ahUW<$BPA5KO`f@skMq_`hC zT_)4H>tnaBX{0Qh{FV`er)SNJjsY>YqU9?s#JTpYDR9T($mnPKD`;V?!dhc7oBRrC z=Lb*nPQz@ux$Jc5J1QNAtV-5=9q;b`KZR^x&2KRaz04dwzA$E@X5LUD>yYEkYfd7J!Vk=$WT@Y!@{L_X?TpL(PO@LRA6 z>}Kn=cus39NMNeJjgW{7SEB9Jq3pYn7b zSNQ^i#9NEfnm37>`rP6%8|md&aKUmf!G8y^kSmJNow=g?$=Mn?%ZF=ZYq8-kgYPKL z6Vd~z3mKC*Q(+N?^i}%HsuLBS4RTJY^PAL5c+32A}~O?qJ2>0{$#%RER@a*X0JhyQW2v%Du=&S76hrWjW;)!E-MA z#YA;nqC>rDAn+U?+91Dzz0#{rga|Pxwvzj-T^};!r@vmOolq_FcX!hm@t-?HScpP(MI@RMwsHcn3v$yLCPw{Zk2{KlVh+Pu>J2_Y@*@VcH5wp+~O zZ}*khCq!Dr`%rBd>0TjUsaomYEtDvYr;HFGk5$mhj+^hrCya!2>T)(Vo&9!~P=X2G zv?YE%zN}p%^>Fielwv4bP9-$hgIUp{J67))g}D(6FX;Goa}o-qjCYO=Ezv6C=HKI` zgxzwOKyPMDA&RXctshL2BOI&UYrc9mVON_{LL{@f(ihF`b!uLo+HC^YYyOBFW}q3m z9#rn^!aY$zH>8mYEBJ;@Oa~k;5T)Bhvm9+Ec*PeejdkancxB`?D#y)#P2#i8U$n6un=a3naL{bF zkK(C8l%Sc$kQY)9H$m)WFK6Bx3&F!_DwK4__0_NokkMZ0yZyy2~ppuF#iV-F~ zu(3BwCqb<+me)Df97t+@LAoe6){G|{kpCbmBSGoQ7s!o0OUbH8@agT%is^&ZdO9uE z=F564s#D*cr~0#z}SW79B$X zWg%WB_^6hC454}Ilv6G50e;fmG@LxVP;ozn9p|&G^1J))wOj$1m);L=me4uv;)%ZI zNQtmf9o^={8(m@PH|sLjT~z(2nR#eHEZKop`8e5_xKAGV2afNlhdAUqN zH}{*IfSa1H#g_CIO7oM|lpohk#BgQ%9@;ajIINmJ>D2i!jhmaG*G0H7RDxEdSI*C2 z#^#-P9LFe!!_5N@)+An!)=%s7($QN~@kC9W%KB0NRu%gb-z?aQLnpJGCEr|(Rr>*4g4O5C{i zl|glpZVAs!nh3lZzwH^;{>0`WO?t4RnnMm6FjO~ zwmGFP9(UO)I^Y51<^)V3XpOw@ci^pMN&m0o26Ga#kO=2|Ru`Z$*Hr_|O(xp$`;5gz zFZw&uYF5Cj%&9HV`uE@e`_6*@6e;ksUpE4QY$A$&BjCS=#VfB`3|6$vtXTUXi6*e) zWVP0GNc~|T;5GgOG7!rH;Lhz*(WUF)T`2_A!9ftBJIZ6gmGR)cCFHZ{CV5bv^u5V< zzWg5wL+f8z6t)s`s)g6T8LkR3IAf|OGQb13{eGwq?*>2i(5tAcBKle%h}_ZdBKMyJ z#dpX|$`h|2Apv;V*_N?787QQlUWpAg7?B~)m1DT=tpak3dT>9{M47*>HT}o)zTF{9 z385d}xima7xL(XNQA2G%L>)(1T5GI6@hqqn5cO(b9W1HgHc}2#{^xxmS$#>;fBPo8nOxUURI*^I)@U4S*__m zZDe;RG^6&(KYgT}ChQNKf2Y#$D8zOC^}Gvsc;0vX8V8P}TFpU>#e9tMe?>>0m4VLx z2Yw#G(8~HkLk1pSJxhODC@9NVT};aMRty_(Kn(l{!T-Bz+^6eJi@_}TaLt06U)gBP z+~PNx>2WH%GW>dI=To-VjInangGZeb^v8K*PIqEjBF5TN-y_z=T}lmeka#>S0q&89 z8c3614=>zVvdeM6b$=}GD$u)ue#5r&B{ZNDF|PJU%ZB2jnfQC)52$GYY&g-myMiJXWn%fh`$1tK}4OYln=J zERUb-uuz-QClLd8*GTs-;&u6QpE(Il3Q70GTQe@eHjSTTn#6HVe(>NnFZVZh__b7C zi}t(LSL+h%L@(n#W`5GLuXwi^@CaOs@JT2)c2`VQRVUVmdOI#nLvgGxNmnJH-79W> zaa$|=NFQV8tg95q>yq#PpCl$_ff*x8x=-m_|YNhNlEv?*~sUOr$bNvoN>fnb3vkz%5ar_t;S)Htg1rr(Q z`^x!6t#lrzr=82|*Y!E;cMWM<%{n30Vj%2_c{rZ&fOAbaGSkdA0ei7W{>;3rO8k7O zMxFFN;LpRP*(@|cW+&-hw*vH#t|0?Ho`QWw2Qfn;00cC=WnPp>bB^>;4g%{j8 zcCr8?Mc?LR;Me*>tHb(JYQJ3tZ;`idG@uo}8<}Rv!XP~SM8X0i9=|J=6su()%2=68 zi?Y>cA)70&?s$By0xh`Pf5F@)ZvabV^wupl$D%>hC8!l+fmmx zJif-}cg{pP*$Cb&4{13Gn+h$NF;2W?-bA4q<|qK4n1;gGvRehU(n%<%JS3N<*GxiU z{-DN_jH-e@erbwQO!wmCjLD1{6%_LZz%lPoHvk^^bpj!No;ypz=nKy-JhRJ?sTnJa zs_mdz^xErsaa1zb_PZ#A0Z5UM8uo9JbIErI6&$fK`XxysMQDh*wA{MXV3-r{^YQ+8 z#>;lIs#4xZW3lt>h!!u-zM*hWL!Yq{nx{c%ES$V_w)i~V=teA)ZlD|3Rl94X;u=Tg zruH{H)vY{0S}MXY%eoV$F)fc~CvYL@$S+v-=!wx!LBAL{`S?dZ(Xg-fLg%Xq7u`>< zL8;tk0(ahEeuqu1M==|O-7SwEs|V7bF;gzXYp8nf{I(a~Q5;iI5A3*{@QiFEcpfauuGm#@jFXcTh*$;@*B-Xzh zHiPBm4b3f-m>mpHig;E3x!z~LmX3WO zB|{A*RsRqa6e%)i2B?#Xh7w;sY=+jy>z#zvhl5hefVRD-#0*~F6Tg}>v~r0Ekp>n2 z{l5y)2jst;0RA$G4J{eaA$X1WO&~+tEd(VkwY>PAG{!L+2{$q2wL(14uvfIxB9+R) zLymG2pFF{#1}lu*11JgY>@@!m8!*1u$@v8np%{A%}giQYmFFo=w^XF%EPQQ|W{tQYWY25Mw>eLe# z5Az0G4V$h!sKDonF*Sl_i6-z{SpMg%$?Zta!~m6|s*B!3opxIER8E`c0_@H1yg%$ zg=^AbqLPR1*-e~NWHwxO5mF+18kU$1Cyk1Is%r%EA-trTBSBx3L8TvwZ5T5tqkc?x zncsuctEGOKS}G}%JzD^dDaUjyuQ2$X4Mgb<6I`iJwd!P+@({;L%mG9+U}AZ}M@0z< z2TK?pS2_7*im^Sb?5OTOd2{`-wAn0+HVHD;r&>89A=Gcu1sd|qSgZ&g^Fv@~Gh)6s zDq~PiP0(GmB8cXm#v|QBh_bj~quPltU*e+(&eGf=DYG5R3*e?F?)IO&r<w5)3bDTU@0 zbQx&jpaH&zNx^&Bkc@6(`QYHFmiLc9-oV&%BXLA_yGfs-FEtd@AGT{NEGNe0s9+Zp zoJ+ha-EwjyLpa~)7B+*ZO~`)W#Fl~fx$8BZV$MtFd)E12{pnI>a=KeY&4l%cKtkf> zkkQ>G1HUUfxGTcp3)r;3r;9%h%j11$IBz(#(o-_97a9DGYW*$5xkyetFwWS3P;O>g zx~BI&5qa^(up!PpPP%kn9yO(1X=Pe5;~U}NNuGf}c5>3fGqm5Gm4OV#)^7$?e>D9) zMDO95S1&c53E4~fh7MouZGzUb6dxiiIk%gZ6qO89&@`I9f!&u6qVJb7DSFI&zjC_K zA^A`z>}+Y2JAdoOUC9VYd7&&+Bi#!v>>Z9eg>N@041LX_V(4@3xA%er45Yr8NVxUZ z1}8F4r7)CnZU;11`IuG;dRG0yH%8QfA4b^hM%dS;ZuKUMqL^!^r}xvX0RBF09~*kMbV#bMDK2bq9w~Xj0QMINA>6^@~@Bu@7Zc7`FhrWMQooaluiYA?EFrDEeU=V P5pwp_MdO0suHN|{hB9=6 literal 0 HcmV?d00001 diff --git a/figures/example/hp_performance_glmnet.png b/figures/example/hp_performance_glmnet.png deleted file mode 100644 index 8a6c09b700052189f66af09e0b8d0717e3075fa5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53143 zcmeFZ2UJsA)Go?79_3g$M^ID*RKS1;C`d0+5fKp(5RiTV0jZ$}2uV;>q$^cwl-`S! zP!pv}mtF#d-b3gi2}#~2oO8z=@4fNg|BgGxegA#q>@gT__ug4+uDQNB*PL_Bxq_Z* zs-8Y^=>!W4%W1Vo4|G^qe!p|@_xrEFlLjfi6yTrZE{_b|Sy)8*4*q`e&H#I{uv}qL zdvNcWchc&(?O%Fx-@k3J|9bUJas9(TUL61N>^GIC4^}GHUg2`}@(S|o9R|>a71`y3 z+T{Zk;V0H!dFrWHzyA@b_(D)P|L!CIQ$OI~4{uJs5a;RNN1(c@7GxI`j!y?t7m`3% z&xfc01OV5UXE+@D;cf1D*8}1JDjz2uK6!l>#JutU+<2UMb4BpK#3vXCrV4F)KWTzZ zbPOlKB>FG(|AW@-Wf<(&Z)`tmIU%(R4WZNHnw;R>L_S^8wBNyC{CW9Zf6cCpnK47` zjtDf>s9xUuEc463n`a3_a?Bf+dqIc1x%+hbw}Uq?2?V>kt}1|hw<%5f=mu<`;_CE8X${jc$QHFJ#wr3KtMhoJ?5!9skc zUe*z|r9L>S|Bahgr{Qh)-Z!m%3mLZgxCEdi|CBSl?nqQ0{2wxPUWpXOTY zi-uJ}o|mNTddsXAFtlpvB0V~X@O30%UnnZGJDD;5H8lUU&wvgW@I|^>h*uqDwHd~w z_}Ocj?_KW;lEt^hlzHIuJ0)Fe*LT+b(q%5SY|b{p7~UdknZ{#jhC)u^7yQPFM2y#8 zA!ixCzUcIuDMI@0p^74mKA0Ade@W1QCAo|clo3#yz-ZGHi)h7_f|fvq?b*Dv^J^|B z!79TDE78iSLY(O8$X~Wrj~vH4rn(i##4kngTNbVFxSjt{IbNPXy5v#6#EDFBILWeK!>Pd}sLEV~tA(L(V>1I~df~mdI$*#X#v!?HlnwReIjNMJc*Bd`v!5 zMaQ;RP#2fmY77d2RZKRUc2yc1$CI}`C!K|)$8FURVc&$Ll{T9V1Y&~(gDtt@H1>%S zXw-Nb9GgeKZK06?VdA^EwDI^20od4w8EO*!=kS0463-2xt;Qi;_xz1j-_QqPGy|ku zHU4e4U$WuExBTNs#ku!+c)T2T%up+NaG{are%g!G9E>q|-%?M>&P*cOpHxR7>+ z^3emv4s!#H4Ho`H?T>F^%twWCpGm(%^t(%q?c~aswMiFC2D?*IJ>QzU7cnUGzV2bj ztiboo3)Ho@-$67l50gJ~Vok8FeVKNxJ`QnW8s>>L1kz680v*0?t=i#BtIgDorf)W> zn(lP<7O=A0r)Nq!_g0!sQwG%I3Z7$~g5HZDi3uSLx5iJ2I0LVNcXI$kKDAHmId2VW zopq3W*mV!l^U;k52PQXmO0nW3B`|vHZ2(%+dh{l z&@1iboOf#nkprISMSwur*{`dRb$b-H3K4d-f(R@DoW@!BT-<7GffutwNhBiW zyxO3`u}}QAv;u1SC&XV6-X|h+VbG=y#`-u6QQu4q{xllKI@M-08ku-DWp#vX}45(_^zH;w$+j90nqNBy2m# z&OY+$lG0IgC$UtKy=3z?OkLDdk(_~iQWmpFi5ko%0yE`v9}zuKAU zHlmCvlNeLEcsNEWI5k@zooAjXyE1+i3@$O)GHOyjyHYMc{WVaDi?QkBG)--!kQlqG zuiH9Z`sEqzp{ato<6G6zww?6JFXl-qeH#9Ql|V59=IJut%V5-BiFays)Q zFFjKS{O18D?#>0VKYO%BB~5Jk?2{(SSx=Gi?bt>AntqAM(WIU2mhPlc@9u6VSUDSP zvXta3pKl?=a0QWSmzu7EgHYaG^3Vw-OE`zhz%HW+hhh-62h{{nh6Qs=3~x#>d|D|; z!+VWsEIyv-Tq7ZEGXN5P8sC`qL&1{Ks*~0~g?ltiTtJ&&Ku0zgb1x4CvM+6ZUj%x)seeM&n6^YNwp@o};0;+B+8&syb7{S2{ zyHWjpqVn&jf$e2?7eDho=iSv_p5*7d6m`Ct ziA4_Xy%4T5MT@AX94cx7qKZX3gX6=M+E$zDyr&Q5syMIlcpGUa&b+zk{!>yVJJo=n zg~J}%t41a63dN+-rvjlx!#^?!qfVt4l<(tzp$QM48ZF0(A&`X8@^eY194gv%eBq0R z^lXvHT|8D}A7Nu-(|SZ2K_~C+(NeNd81q&>^*-wQFRo42GoNlp-GC|`^n2I(J$tkR zp)qJ{h6LMQHC%&5O?xhll)e_bFludD11(zYUmhH{83=?|dpbOwl2~ab78;`qUxOZM zb9fSZ(x#D1%L6!@@8LQ#jd6Ucb4>)PrJ*8IflPp)XKA`w6|OZWGqv!~TZIpLV|kV& zbFWJOra2GRI7>I{e2g48$95@4 zz!-(yvIhFHvRa7H)#qeurAyMzm$5ZAoe8}UnalC&*8}6ykKsg`NTuwc%^z1Q;1pSk z7y9#cZEaCK*}7Sp>b~}CG zA+UU|SCG9|RT09Ds~gDv^>tIwjT?M?R#5Y=Zp2(>f7zE^U%TlQVLFdUk;&wk+Pibz zpT%#W`Jrz)I5<|<)R`Y~Y#D*t20q^l5FD-k?cN+iyAJsdYwKhXkt_XjM6O`I(DbY8 zj8Tmj)3)^&KbVOcmFt7~CRMV*XN6r~#Rwa$%?tfmq)$6&ejluwgat!K6Fge%hTVeS()!NE=OlFiFx`b0Q20DEUkz= zTKr5)QBmX4(U}&lu}%p~uFh%?yFajC>&+L!4-baVg0}@eb{IlK(kjO_crp7AZE^z8 z=%v0m{JMUY;*F_EWlA$c!%TLcP*S4kQA8L-7rRIu{IFO8FpDOCvTHpG!FAFmmlinn z*89sLbXv?gn5Cv;jMsip;Ppm=7<_IdLTWAEcpyg?gn%}@I}Mv>4(eM(g3P_6{R+En zA;v8P2xXNdjfvHM#qR!a+_hcZfz)3n*gL%%5?*)_jiPtscxg5fV|Fz%C5|Is72UWF zD41-~Ncwy&cc*4EHzmlGHy^RHyhWE1F6xLn5nJm;~|iQpv{+0H-rIs_8G>+2Jbl~z--itISeFIuXDE8$NU*0RhPljYx=?kuj+mXd>aLi2w#rt~q2P`BF@y`i7lr6PQP~0x`x~vJEXo~-Z6lHIlvc19<>s0t-&le7XJM1|r+AGv_F825L_iFw%bmzkMPf$Z1 zH?mnd>g$Y>$a9Gu0w@zhdytc*rx}u$;yalUP{V;yX!V)q@~a^%btj80 z30Lksvht|^rPO++L36}j-=C+`1SKTsh9TKzb-pl!~V->QnX6}#$IrLy%r+GwRZg_Cq8#M&ni(#Hcg8Y3f8LZ_>mdl9$zI5BW*BOj|3 zZttC+b|G1{v>!!cGW4HaB0B6)i|T`s+#QJeBgC*i*7R(46PN>NBM%%0?hp(Do`+MZU4T5b9=dl(E z@nEO}THf()pxik{S-`}*;@pEIrPR4N;#`eCz(&RmrKHK6OSh;ti z_^4$$HBle0R@hnT&sXQKuI|W^j~X)~bn3oi2#J}iV*&>Z_2-pveXZ!c4;kfEcwx5T zjG`_eeYOyQX4HFjPqVqVY3ZUNmzy-`de&f$ah6%Y!?5MSn^c0y0c%)buaB0DG+655 zz1_{m5DgP`F+p`-jNFXcckM(-yFgS3xbkJcr^IgcSZQ^KO zpZF?>35WF~*8l(?jRItL20;_0_T`z2whB4?H4&+e#9H3P=HbmQV<&tYqKI`>=F>bW zKtKw(@X+HW&-3@S{{Wm3r%PToVc&~@JC%XuiWTJ8Z5oxw&m|&UgeyZb+aT`>Oygnm z=38CN-fACPAXdW5bV*Vr-* zv~O%$J9Ub`Aoe5WQy39b-6lUv<|k5qKcwU2VAAKaXJIKm1UtA+|;=}q0)Jg{aqX|F)M?R;SZ%kw=)n%2iJ^#GTR!m z@!P9_W4lh~_y$?#?;X|u33+WA9mc%=i-fwE)ct2@VMe?Bh#iU2#F+@N=tSM*>ytV< zI=hF#E|$HIYVVsYVX!lP*i&3mj_Z@(n`+c!Z`$>wHkl={F?#`vo2?v1x&B{*TmOmp z*MEUe^ni6_kLSJ96O^RWTCZN-P`*}n@?f!R$Qe@-Bq6CdYc00aaxK)SU0i%H-xMoU zqMuxrNnOQNz!M*ytFgG(wXEX4&|#kNF}_Yo?D69qQ}mRQlYAK-tt}GIjDY&{zR%?b zS;qHGP=?$g5eelt#OO0`62j!!WOgFewU(dRLazt!Y=M22G8hOci?xJW0-HeQofVID z|30zW^~viQ$;mZ)Awz+6yWc;_UQG0o(XYD~s2m|n7?Yr&A5VmBF?|Go#cM!_mOzkQ z72vYP`Vrxb`09!>6xPV7TOEQWMDNzjQ0cGrW>To_GZ0Nv-KWi;XHom~Idh)py&pT( z!M=-yAXQP^ReQ&@hkXr&bee`Lu{7_y-Egr*04tm5gz488Y9Z%EyfzLM$GA8+p1!@Wzi%`yGl%HR=}R*qzbx1(0Q) zDr2XD^3{M@c=VM;*wZ}R$nssc; zz13=U3EExNb0@RHc)_-w2gU_`vnPUH_;6emvZ`wvH-D0eNZ7M;VeGUz}7$@T|z zYn`||~Z2mLmFr15&ytqdQNN9NVWcTbT2(vE**KS&K# z7gNg7T9FHKRX!ngGf~!JI}7DhXjo59v^@pij7k;P&RX%oR*_eoPfv4x8cy2S2_doh zSU>dQP$>!3Liufc3MgDncwQgFBxe%#<*{g5lDz9;dmlP~?8DFPH^vaKyF>i@RKYJ^ zS*2^0b9d(Khb{dWF}sfw16g3`7ebKUr~|N*?0XE^>hm{OHb%+u0Okk}%w!Ynk+e zXQwO60quC5RI`{`C~)(}$FZ5|kTXL2eglCJ&vpq^i((#$BrpH_c)x%{Bc&%f* zO_*5=_dxol0l>|2FZ&)FXXlG-F0-@q98QA;Xo+)&s_EApUsubK`lBl5GiXA8PUz9^ zQF2gwgUTw4DYURKGD}4lBkG6MssAN#TSElqVwd1GiWQXXK#6Dct6uw>`NZ&Pi#rXdl|tDF1NGh_$1uY>t} zUKCCuMm(QnIAn_SLRTLb=(NO26!I!eLB+aSgHq>s85^19S*9-Yw^^a~`trAfQ#B0L zZNsSRbDR?iwQ|7#ihC@E_LnP9s6>yAjrE$b$3#U%efo4krDwh~QFw=)8Q{FD@~ldx z#C4`QwXj7ODYJlUOFsuslfU@*Xh(k-rGpkypws?qPM7!vI2=tjI`_29sNH$y zH#`r&Q_7t=D;P@C^@XSv$aw$d(q1c zQvB}Ha?x)N4HdA0-WqPTc0Uy@r!wFW9c5$RpRMg@g$PtPV}5qtedD3_o&< z4MohO4NTxfb=^UZXKsh8`wycSnA^2A*B}jYVjE8)%DndegRt~V)?qiPK*&U;oULy9 z-Hh?t-iqdu2VG-9vwFU&>gYlAbUHc0fZJ=+tKGv^3%=V0GoxMy0Y%st%96L6@(1t6+MyJsGgY1&*~|67%DWaiS-r~6N3!o>RF@CFe#8PXe*HYIGAqN zb234{(8O?e7>KKII%nK*GHP>Y%WxPiv)k|rxE|KtcK0-wv?wLuIojED?A37p$FxDm zk^I4xN~^>eio%2Y?K7H-X5M!k#)j=m+Pcm&8^50ejLs-8R^){AR;-Az>EKVjrS@R6 zmHw~Rd5`fqDtU1BkcyeuLD8N;0g=tEX}o0lChqOZN$JE22ZNe?_4EoSV-Dr7=S9qn zTU(g{fxEZ#L^U6Dj+O}-^-d)kmP*Ab-GU5ukl!Yj8=>uVmbe5e#)~|Ayu9?i43)P^ zOWjUN>#j6{Q$Hp~>6RIm+H__=k&Fsqot=v*O<=|_SXLGF#5DQ6z{RRlRwsOPuWJaG zYdWS#*)3;^BucWaj+fUS%~b!H4<)9a%aY*i=oYk^eN8?YEHz_(?3_wYPL4{VgZN-H zye72Zh>or>?d%Qa$bZjRZVD2tB3&IniAP71$*fk=u?qGL!)y=F7t%lU=Ru0xV&g;& zafm=E;jtt|`~KkGcjr1TNoKyJEoF{ZONGlnD>B?leOoC3()eW1{P=7MSKi#X07{jkY00Hd#C9h2bSp{oL0hg?>7tNcj#FGQmw4cA*jesC$9>=7 zBPwQMbH9XEw4i!D)d$~q`pubJiK}1IEuJkv))^09KWlXQ((qOKU*&~$)tm(nOJ>Jy z8R*n#1?>-XJQAB|Rv9(6@=a5Kjj2@qiLY{jo@WD&p$4qm8!WQeNdM#lumdp`hBtA= zU8)hj#A^&*DTf8tCSvB&(s>q=`U=t)u5ZR^UNn3$x~Vg9W?->WUDLi|F6?Q18pCaY z-8>yqdf|a1Z68WxU~O&BBc5mBt3c%=yEN_A!2Rt{sdJ)IPY|2CRD5D$x#?%Y=hCpf z5ArGTI(xrAs#gM~^wzA-p_#1t3zCaLgNmVd6qhK%powjCkfAn5`#uFP%~+iYppe#n z=4(cmJ9ZhQfkZ`v&+Sl6sCiagtMy@L@Og}x@ooTxW3VkngGvj!4Tp`pvZ zH7FXdXfZacVtCZ};q^PRluYwe!xjbaFO45Fhi|`uDLoc@pZZdlvIufcSoe(!II}ud z?K+;ItvSC5v3-(s%fWi1fp@?-*_UAGl=s4oiRj6|)ZpfzY@^Z{=PE0LBI$H;%Ege> z;dd%CV3RwDnyaU)&G&1}*&J)x)h$}o?UB0#?6?c*2&AJm9I!(Zs?2M_q!0q@fFpc0 z3~J)}S5W)&A;ran8c6UOYtuC={kOAhTJc3v>8|1nC?!c0}DpDCU=)P#5kJxR?6 z+R?pBV_*fZZ(IyNuf0XtJhTr^=UJ(CR)Z#aq#H4svnCoFB(l6)Jxgz4^V>2PtCg@B z%ylAi2n;&GhJ_ZA8GTBf4za!n%hF%4zS&k-s2O~wt~)bhVYPDY!TrY}W~wRP%@;u) z4$+nJ*AF8Fu&4KL>bG5L*Hd+D_!{4bF|4-V89l8{ICh$=eDP9+dG%VF9O{qkCNi|PM6a$r%d^Xx(2yt8pUWMx zo7ng~&ke%3`2Q|%;=h33_&<~8@xMxLhEb)<)I`cisEhTeX3XlUr;um#vkT7*)xnyU5L8U@yrX2UELf>P-^Q{jk$tyL)po*CEOVovnltI{jXQ2l?H zJM+KF)C1D~Q%O4ib_8(h|JeA>|4upm7gGOGvfICM^sgNKD~$jK{uPn~Mfv}YIroiG z3MIPg4|z_vU~P}Rv+)0Z@7+C)s^ej=Ki&GsxqkLTmA&(Qdn2=4y&;PA+;UN(6Q{eb zbZ6Vw_T^kd`;3^(Ai?6)&(r5VIGnh4rQUyR{)7aTbs;qR)-8!Z(nV-WH&$sOsoPJ9 zcL!`9K_oHlnK%FB8UMGG;D55E@BbqY|1C-SpB&*Y|D;d&42LIqAIr++GtH|22#a04V_xxrIO zQvcK&{3}ZTsu%zNf+{yxw~+1UnLGN}Dwl~!y_Fl%jvAgT^F#&7BLnaQeM97 zQG#C|U(efOvujs74BYW9e{)Us?Wr?B5(}#!(wLRF=F#$^C$u?(6y$C_#E-D8e4@12 zprP7!t#V@?+%;aD*il@!+)0+;S%gm4%sdbVatzDh%AVFNk)q?aZVbCf)!B6KEP zDGC%L`N1Fm?!SE)d(tMXru38D zV9ySJ9h$eH*uOLY+In8MTAr}&;^N~Y0d?lMY^21m zm~T|!=hO~0+SpWIfrjcdL{R=B#flYkUgFe(`>FVv_|`Y_Z6+o&a|c-ncLQtwsM!cR zx0H1;@->3yD=qEGI70E#JU;|di>xiNREaEG?+MHmP}k24WQ~b~sz;{JcXDQG#mmi% z2;>dwJ39-mYLDgNKTHuB-(1->WgHoUw7_9{Z(5)D$ShER$EHzkf4D*ZzJF@^!CZo5 zJeAKkE>=ojQg1X18kw&d$)96XYU3Wc{jx{1-3%^CtIp{EPGcLbUDpEBzjK!k^-I&t zuwQJp-l1c=Q9D10YlVAz*5(RZ>(j9nZ>wb^ls;lcg>U1w4a@!s!hM&^{(m!qp3E91TIb7*#r5?*% zkNxX0b~xfJG#D`xTw% zCIZVmW+q(Ti?lNNJz!;ddA=(dmn!-9K7&W)$&!_235e%oInN2@gr%`6_de9Xpro`r zXRbqn+e5giX(WHsxi}N(_yXAIc6R!e7=AUgA*F`qWZD2%q0-g|k6&}*p+m`9dW4;0 z4w;HlKp|3ac!Kv6^!b#-R42FehNhrop15IsYyr7yXGcT&frseg*jUHB^y5hWC9iF) z-_mTrgsF0HTX)_5>X#+PDZNr2KgV&iLag6%f5sB&{<(<6TQ+QLub8uSx26YRoW)Rm zhf7wVpjSa*q22wV`oaRdpju=kyCxSpQCL`qmELEU*1MGEI;HOJ097OqB}Ef9_ky=8 zx8DS*&Z5^n4kuoMd#x*|&@}Nz!=Q+9R610`9Svj~hk$YFWYdrnk^H98a=cPs{Yc!| zi|Zc)qGUq=-Bf|y9bIV+QYMw{Rd8l?Jx}Bz@q%54&G5QCv#(Eze!64VD~j}8|IR8_ zv--8Ux!II{sA-%I09~YHog8&^M=g>c*wEC7{H3$Qw5CE_@x~2dT&Fu*je)*?EdNp# z*JDbw6ba(X=YhTSi#{XO{8P$&?*%<&w3!ylFw5I?=>V z+Z?@+h(;^O?ksILjga}SCp{mWPGjQZ?itTYnUCAtGR5k?l^#>|gc{Uyw99v(7HH^~ zBh)hQwEgb^2|RYgUwzcwbyv`iT?#or>soktd!B=>mbaE(Yd^icu}3rV6K8V!M@Bd& zXj3&@?i%gW{qc6O4xCf+uoCfY70&BaO=#|go=s!!NX)r&tQmn^<}gpVgoHV4r@W&$ z-%*S0e!h?2a_u|rhSap{JbeXotcN>!Cd=Ng*M#zA>|ape>RgP4k0S#$DSDFBj~(S_ zj;bQof4m-Cy|eczjhY!4V$WFV2{xlBcLFl@rl&ye za3!353g~dPh>|QGyn9RWd@GLVZOsV<4|UaT?T$#wH&eA_2%qYqS~-ZNwH1jqT<4Js z0a{A%#&F-Cm6K4kV!5k<`z3w)Qbk~|)3pJHGU$}k?rK<(0%f=jz+WSPKWTYpIy=kg zm+%Xq$od}C-kya#9gUbtWTUkVkVFdY{T7Cr3r;dzc7UXUs+qjkd|h%Y>!yk+PqE8$ zAo2NvkxtwfU*zm-UFw=QUw<fy3Of*3(_F9ndUe;HCPKQ57-AeJ z%&AD~2oaXbB1SygiFYAq%p_n>*UT^FNc0gC9-mIUg+X$iIo!am>b*DfAaaPrXJEFo z3&oOfWG=3#^b-4~Bq8DX{f6ksj|)f|UQ^I%rtkRCz$9{5kv@EZLgj?mJ&B~8R=1&f zw&o*~`mMjInPZkCwL7zdUbJ@r8cAa=;*Q;+kzXx$VanET2@LkySU$QHTWDlJ_R{Mu zc)^&Uy$cs4;ykqsQ}<>a=zg1K#ApNY%}_COQVG(u^d&t~&p}w)aRf+e-ax>QU6!+a ziK?%=d8JO1Oy|`aS>a|%4+@d)~6g@;;~ z#pCAx&F^FR*Qxm*IyLGiBf9>{1psRE|8<1_#mxT@GymcQSWN!K$-g*Zx$vjjP8jO9y8YKg_5w!o8d$#N z&|q^DHZJnCH`#6w+n3S3zWL5?tzTwBZm89;JZ+bLZ*;KNHF#P4BhGYI=h+fBJ#=-hK#3YMixXBO)$^++bjj ztFR>ObAb^{C@CAI^C=#!Zcjk8=R6Wq_Qo|h(k zH=ET|L#y}bF+>v+*Kb*f?E=@MhH{5RIUkHig4xx+;)SFc^Cy+J@Z?nAC0>kz*W?Fw zuzTltsewLmDsr`XIi{3|nU?^axp-%{og`A=vkox&F2`+n%`d~kq(@0oH z+%_eo&kX5mbn?9LB}5N_ZA`)EDLo~NnWD%-yf^|Y&^5}s7ll-!j>wA@3aXVOK0Z@x z5`Zd6P#2 z;I4-WfmhJGBT>hF>K>w?gd+=Z2XY6MSVw#nnAzd&HMLxTgG$I^jj_rtm!&sm`?_RS ztgP79s*Ab~wbbkc2H~VL^ovz@mF23e0l%8^fHBgz)`J|Z=P(G#%`F^{^;qsL&#p8u z#L|zPxhR#c)uZI|?ogOtLZ;U|eh4p1Y}4THFknaJLlh`()*g=epMA;yXyK5vw)zOf zB|^}3!xXUA9CT@-^aVCDGO{<|3^(n&wlWOb+ML5|*MC<~*wwN%Kze*T(DCMhxO*%q zI(DepntP07*up>OyASFgDz4E$uRWuzQO^)m<%&G&lEhz=dqoPxmd1q%O=`v3gzAbSvBK1WAH0-wC5S zCd62NhQAQdOmgk6sHlbKxEgm30+m~k=QQ-utp)5ISQ{J*TdW$8)Br<#mRGXOUAOlW z!^Iq_hkHr>CJt;xo{v3mteWZgew+MgrA2ORGOt-;bDwxlUKJB7T#RXZ^3r=JHqRN( z7pHni+Sd%l#vo|xir&VEv5nG3)N!gUCI>cM&6Q&RV=n%XL#r?2T(1EU$R)So3=(~{ zgC@=!2VVa+X2QeFFlPA?zRLxQL$WM4?WI0zggA`qwg_wz4xEkUm*>R>OLSB8qZ=d=%raC}Jw zB6l}$cARo>aB%lI6q+ksqp+lx`9Qhf#klC`*}v=X?n;%BWHy?=TH|^fo)ww>9oDE_ zDBqGP@uY0qO3cs$j?FD>-ek56d`*Y^V_%nc{3x^Q@L3vW~wOR z&gzv_0tzKW_iZs>vW)yRn|UxWjrZGaKw%&n^X^4j`zwZu?mkzT@2|*@jyL?S2T`EK z!H5*H1{k@zQ=KWM8;G1jj?`?)RJAm)O?KOKJ9fpu+##B->?+YeQF7Dv+<$VYAdTvb z4TSWbB`XSw44~c!E{CQC=cYb-pEs){MRNY|qO@wai%uVYQSd@4o?D`{w1!CHHN(e~ zyGTA5yM|T7ypK{GFTUiW?ICa*)lKSyE+T0oM|hj|*G3b9HGm@H=h)0jm#&WZt=OWx ziTO84@FEPU*R&5aeECq>_5Ts)V%Zf1h=sR;D961Tr7Su=+C0=5^|}2J?rO2Q$?R9( z{4u&Cx4(?Ukk&8K{w{s@*XL7d2-#w?qqhc_Rd)&G48Vvf;W8SPT9wJoTr($r^OFm6oZ5LlSXN)Ju`hn=G%jC~wD z>beHP-bmZ$_o-Rk1lZpzGiTiEsRtUZNk}CH;AFm5tvb_Z6Jt?_KcBw&wmv8!QdaEX z9Xtx@BJ|DzLc*d-lovCq*u(*@Jb%0Wr{m>Pap9VXRxU*EBk(bg5GHU(Y> zlC{Y>OJRX$N9stjl-Z4$7L-?g94qw6{OaKd8(+S2NzHt*dy-8n;m*>lEWEP^H8Vd2 z#^cj#H>1Ucoc^LLk5*saKK%O4W6Oh<9UKIE_8tdKu$r%rGq{nXLua@q1QdOA4`(X! z`thj;oe73w3cyxFI%^8TmNogT@#X!Qnj$%Y4fjI^CaC|~5R@QUnu7EO&Lm`0=zE#< zz^1z{Jlx`|?1OP6s8q?iwyUk?|$1 zU9{K;W24`q_c2Li(b2hl(69sw>Cy`)DXwqJhrKy#$9)VzUa?K_92}X6xX`20Darh} zOwIK;NT0sYg&&hzdSyFdXn1jB$16FI8_{NHo4gYi)pBO-)A@)$4|755v8IJR$GfOY91eeBGZ<}3dWmX^H$2o;w= zoBC_lSZp|u7|TekTq?=`NQ=4ef&_0tZZOdOaneQ$FJl*tg--{8mQg}CZd9*+ZCBcx z46sRJ+IeNyfmSq=${Z;GGJ zVIgL%dga|2Zr>giP7!lVEuGv39GwWk_~7BBWT5(>Ek-!e_1(ePe#$l48fUFeq-oX7 zhGV-0&FH&n(!dEV<6a|$ZK?JHNEBvfT9<7R51Z}b1WZE4&537RaUZYJ&3uI^RT9h#x-lUk;a#^DN{N!RLg#EsE}4qLmFa~Yn!wE`&Aef|3N+moCR-8UVI>JG|2 z2J#Hdy{Fy|nZXEWc70D=eWtIUf~1qt6TS{fHjnS;o7GV=6P@jQjm?m>9U<+da*A(_&s@RA>yLM{8|k$d~?5D zwB{xSG8g5?2Htl8yb)czXSX7W-;(Jt{5+pd*n{DZ>y&IJXMwa5Jn+h59VWVcKV%7V zXXN(YY-A;SjS8s}aKiFAeGW4`h`p&kEpQ!u3Siyl)>fjcQqV3i4dJ|CrJl5dGcD#j z;(;rOCwm|-J3AZi>##N7d0P3c=gQ!K^lQh_vYgb^)V0do++2$mPc=0)+6DkD;D6UO zlVqT4W0_t2^71eMepOXfx5t>%D$>;$#NG|e5+%@KFu%3E-JzLW=FthTTCd8D(ia8N zs3uNv+7#wEuciYUB%C(72VlmbzVf4Kq-;$^ZoymjBdlkhXjW~QGblq6T{3*fkEo>O z=jYef`n9xlx{aQh_==!WmX{wgRC;bANHzE{9z{98g8TgWbJfBD$nRPn2TeOr&%S@r zwp_;$id#jt31CfSX-llv5Bqk+Mp|xFA7x+w@xmaT(W5PzcHxG-!7*fLdePL5AQP{CIi5xLw%Mfq+((uUWLPcM=eP+FtYUjXO`AE&DJ z+#D=4rB1+zbpT><3aGd^gW}|S0msT6;*u~-q(0Kl`4c#nYaF(n6Bnwq6N!IVI?nFr z3e1$ln$XV#fb3c-fE^6gxdG(v?v6IJ(uRJ<9I(Q_g0H+js33K8?Kq~ALVq!R4Ft-9 zQ+IoKTS#TYg=0W9?g5t#%zUhMAyOhD&0Q&gq;)3Ax&yetDe$_+H+_3`q_3~f>H>3e zbqwv_u^YpkYqE|i^QCNUY`4&5(F$+=IEI+v3gq|nE%R##WP_{;`CYfehDXfA0Mq~n z8(@dw(%9qd*Y`K$u#Wb)T_Tb}1riww*SF4#npAobMu#*1bd6U54#Q*6X=pZ)GLeII z0zFYf2u0%uQ%*iA1Rm_q7*cI zxT@R*X6yr;s8NHEyP%k;m>9jMPm4R%0mWoi_EOuz!lD5QD9kda+&uac0GzAm{ zet%w?U9k;Fq%9SSCWvIr(&pU@{6AX667A3nN1+5 z1T-E&mGt|XnhFqg4#$7qeF4ZjvryJ8G6zjhJRKPs0j75RddUSPKkqEfWJ@mQoMEm^ z{pK_-yO8AGa9X+}SstxMymaT~&*`R+GzD1^5g>)sxim;Om`lY7#&2ONDk@em<($J|8U7Bs@&qWaoBZ|) z;6cdd{x3(4JxP|^=~Wj?R`7AVvtSNr=@wq(KIf+^YQ1$Zl5L95e2jrv)k!4A^w=7J=Me!xn+GNmg(r zfV}0Di!v^g!bP>c`B5jXzNw}Yl0{1eZ``;cDw-ncpbs3)aVl-~^XnR5kpm76Jk1_r z3LTe}mX>a&9su+6X?D%Kzh9l5n-h6nJoy#S*R)+67s4F##nfwLF2SLY444+Cq;c~0 z_BI!!cH@&(c-8)9J1`^FovnaEZ@KK=5hguR*}?2Z99zHlF($?M{5*+-@<-iF2 zS^I{4T^1nf;FO5nsm5PeFBb>IPJU4i++7~XQ%brZ{^I?M*Zt8^QE9IaVq5+;pd^;h z$Ip$OJbzPAL?p~9dQrGC@YKa#U|xoa2X3c-|Meu6D_;eLD_3p;l2%336U z--c^zK%Hbc1g~_+$1B#qgIHcBr0cmrupQS9CFJF~H|#8pfhU}SmRM5xftFq+Jo{ef zy)g|q+%30)nJ@ffRb+m-EAg`V#2T(1S?0%dh_Dh4}dR0QLY^!ihCN^*N4J-mZ9+S~SUX z+wwt6IJXs60QlBlN*U-iFZHm=o#O!Q>^cRnVT(V%047)v4RO$*k@(baEKUZG^<0L^ zT{t;74o79A2t0KZo)!-Ql#X?K?78#j&!0Qzq}#}i6TDFgV9H-g9mw|9Ztds@t?ala zdu~itRu(|s;k&^vbAjy27z9BiK1H)`@Z%+e=1des6@gi8(9;_Aq zjSl3g2Mh*zt%=b$7RBQ4q+Ve>VlzuD`zdhn_+) z=Puno)BIuoGsou!Mn=`n%08Y>WyGa1=nF*F1I-gG3GLy)tlqgf9qtqjFcUb7(|3EZ z1pehMb5^dnMqbp67J<_?s63xGiRcM9a(=p8PXkVEjTlt&dVFpED2tE~Ft&k#fhOkF z^UNjUvg}u&?zIGe5H|pX%5JK$nvGa9yN{yJB}CugW>?axPJ5iU^uJ`tqeicuh=#+c-a4X=STmWH3 zMAWZ&H;+`fL{s0UFw0*Z)=fT)Oyz(J)~K}A78x^xAR-b8AEpdtte zs0c`JN{4{-8j&U)X@SstPiP^ee>=hR|G)2h-}&bI=H9t;@65>zGntS)&wln^dzIh% zt+jz8Z1(VBPo_q)gRP)Bf;RCfckVk&6j7EyOqdPpU-@iArA%tzR?c9}4@1rc0~wZu z?yLgadZ(r;0m8KNO_>o63O89kk*ew&gPwQ1ldGF#OXfyREvF( z?3DZLqo7?GC`Ar_a)K`KJ!yP9jM#((Lg{fYFR#N~Mv(v;6 zuDuJIDVR3|*S%)6dSesC8}QK3Fs>&R;vbz^bOinF+qb`WP>&&SZ+}Yc_Pj>PZZFTX z{s=KB9(?nw?Bzj-djYUUbO}yjS6!#H*Q!ervy$rw5#LyI%#t=&tBSkdgW~{1LhD{Z z$7{R*rW66h3aVsjWrbTC(WgIrh{9Tv&`df&? z$j5{u=#Y>j^oL^=K1?l=v_s@_*HEIA{C1iHQIg`x+nv4>o|FBQrC{z8kaL71XyDa@ z4kTh_n(l`6wQ9#aR_P^5`S*_3^~KcUEW1g*GcNqbxMxe>f;2`vhPS0DFYwvW+2NAY zkb70lS1VSc4NkDLi(Z>nS7*Q#RHNhK<2h$&YoO5nY0=qyuDZKU`>f(_?;T`9HD!TK z9#yd%W^`#4-;&Jb)A1{arv%p{H&*4|7W|y|PV%poHKcTEBm)GM19p_+1*j5e5s+0@ zCKjkkSI`QHw}aT=6V9aU2q)ZXbCZ*dJCZX(bWMw2-jFzS#U<_}MB(P%H_vC;{sGkK ziM&Z}3FpV&RffV~Ooo4govz?>+xVHLfkCAYesfme83~;7P`bK0_vAR2`U|wQdE?t% z;~co)pAi3(`~m`YneEq>Lo2L0<8<}(PB!eRC~E`o-LqaJ36h7o_d`uEyZu%j7W%bW60s_EKNwlVj!d|CjNCA6m10)UG!v?>IlbI3iqpO2*UK(9rOj zI51ucOu!aFtozOfo}8PT0}cuO4)<8WMIj??t*v__oW3ZXaM?&|jYqgf$_-_N=`^jf zT?2L$>5wyeb}V`y{{v$;%*Np)ZU{u06+8y8ohquT7My%QRCtvFU#y`-cW7omCHA6l z!jQzu2Q%_CfUfb*YS$)mnRxSZ-(mtFkhekq+29!i(PF}7X{Mvfop|Wd@!qrbF#pwO z6V$ax;Tor%j4$sFLKKeNd&Yp<5getDT+874k6?`?N|)wDjg!GOdHem49u!^qo!gjd zTn?h(^6n?|v2l5jQRTv?V7o~NV#Z|AKa^FJ4owM&t06Xe)V|uO=m{p1(IrST@0~X+ za%0~He;C~R44G2nm)DDE_V_V@Z~-A-!=MnyPMv3)CL zGWU31%)h4Ad5}`_SQS0zg7YmF*x!v)zgIP|@f))$-ucx}gcE;#_0TXgsgVx;1!!v9xEN!Dp&~IBT zgb8^yE^#zbIpH}np)KbPOQLIsP;JJ9%k70mb#|a}c!9|Pe&DZb#aRN=dNir)xp>cbV7UkB%WZuZ zJfDh68TZa=WoJ$hXgDFLY-wp3_J%EWCN&7nmGl{`qztyR+!i6)mF5Y79KSqpwIYa_ z3-whshVF%(;{$5hy0G3+UA=3;Y^C33MI~fJb1*d* zRcP+Y>AxN;+`apXX%_6*EjIkuoa?aiAvBjwI5dmcs#_P)KuuUZ&&l$GYN#EIG|E+QvO{Vv>n@u!p1 zAzu49v9)yP(eV$$BETr-2aO2U0ng=AWNhs4yZ`9=_@9WGbV$4#cIMa*A~?^-8a>)p zXr=cWxr4I9{pDqNh_}O+R1{G#0*yFiwvHn0C`DXD5dxEBp%-720n;wgnms8bEF9tI zltmAxkwSF8b9evo8nv}aG7iH%ZpN;dMOkkcp6ahXQ!|KLO!q)|qTaD8es*o)gUR74 zomhtF8Pmi1;P@_{la9V6BQbTK;)jO!Llk=d2fymv*NYMf_c*tr8lF=~kGu{&>0%PT zR_h4gchq!rZTrR!X^x#IkX3PsX+nNGi?7_6ItKU6Hgi0bTJ@qH!9U^L%8sLyo39LC zP?vdf+ngVi(_B%c8Nr?^K0;>bDx8}r9~(igy&ACIrLB=)vCZhqStCJ$A`{(+!IApl z%5J|@#S`$oN0-*Nyk{--^YA5KYBHwWq>|RRc+ejvE5WEkMqC&?5(Dpy7KI%L5%=A@ zAQ>-PCR{>ypBm3infapTRx*9D(lA)T`2dtShO!kTU-7K_ps>5#)L=DY19hL}S~kmh z+cp)wXa|oEEWCmCo2OpdR^OJ{2T5_GYw<66 z7wylCxL_jdigC=({__0sg@py*oQR}SC|9W7h>9A@n>@!02%l*$`Pyk{3$tbbP!k)+ zU7WH)LIComN8eT-FNmI5yH4cZn2`s6((S5x*Niqca@++4 zMMnZYUivDxk~uev<%~u;J#y^^CMSyx*eVdm4BZU|yB|PAFe-(Cfx(xsJ)ALko^B_! zlvLgT6Q{t`^W_K{B%7T;ldwQd;P;;wDyC0!zP~eoX5O;Ud=`aIG;^q6{`J4Ob7qfj zfV{;@jNke^jI!0dJ=`v5Qv6JPvOj(w!2*8x{Ll2|*5j?_R{-FG;|`!uX}=j+6~cC}`ZR`Q=e z;mhCwz!3oO*{82oysB2GJtHlgdwUUhwmg-hc89d!{e)Cp^NgGFYfrS@~_(-#)|}K$#%?y02fi@w1s#5x>PE zQ%C67s^)v@4(6b}zs?s*o0Fd7e7p%+q{(JB{+kaj^-3L;+s##u7=ztL@Ua+sBBjF+ z#WO`S5v{(nMQXeO%lc-!`c3$mVm5gb8g}#l;!UnTnr;Yg^6}GBVeh=+HMzndYwr;X24!2Xii`_|6Ho-|kU{q3knXjtT(&$@qRVPiJ zF@`!*wK9=6x|uTa%cBn8?-68mhq^QRni+LN2Je1|H_wkswwzkuclfh~!*HAAjd_lI zLz8bJ`3*v2-m@$tzPI5Cx0}wD&Lwvwmlk?<#(ga1%QwCJ+B{aogln}unyfRO*)wP&KRH@vrM+iUP(I3F4F{_!= zXYtW7#9bByxDjIn_h9DMyxX0vOaz`1qE^alZ@N`aHNjPz_-z#{Rl$m|Y^Avq6>A|z zl-h4{+SV~6(4X(?#;V=hMvcq-+%|go1oUFo?8QeajDe?l?@zkWUIbB^9O$W-9|U>@ zhc6vBFu-!&0&Yvw6LpY_mBSM;7;h`p>%T0!i{VwI*(t8FORNh?@(V>~X5#pCEN{Ta z>~iW<(->TGnd$a|e*Y`y0jL4sySerlegx1PrlutK9tMd2<%?noi{rJ#!3I&g z!GJ>5NccmeM-{upxo)BDd15t`RRJTp@r8Gi#MQA2pHya7-MWSuDa2@-Io<$WVL4fB z!WeUy&8;6T-J{sqZ&g@kF2r_i?)f)I?pJ{w@xB*Cpzc*2X?1U*gLwu;`PXO&p+d+# z;B}N*b?32ORRY!*Mm0JXD25j&f$Ad%Y;*$vC8w{CY+4-TxR=w2Aq^7=(B-HjL* zevouV@en)J8$H(+8DWER+RdFHwENyDKkIQy+Wm!0MhQM0CpYwMSZ9!%h#p_$QD{yg zuEtV&IYoH`Ha#X{HKCJrYWYnCi5uN8b|lv`wCgr?`<>12o1tz4>;`06j&h1Mim7a~ zj{?$Vun()Wvk%A#HhpVQTQyxb>E{825-1#GOn%k?b&onz>tq7{qn4 z=JB9z0n?f+pY|v9KR-xUQ1eWbnGZX4F!2Vcg~s@t*>SM_t8le}>bTT?Z)ENJ0Vy%x z9G7Y^&pnrDb~EMcJ!gqyfywU8en?Y_quQZV(wu5HzF~^GTVJ?%J@6xyv^pDVHeNBO zIUay{7ttPgx3oRR!Qd_Hw@q+T+hV_?ueVj8moD(&e1^ZV#+_2PkCTTrMK}0ePzGx9 z6WoUy#bM^{Mj)kW=))%x`m(o#@Jj9@P^wadguPDrT}sf~_B+5qEEA-8x&Ou*{iB7a z5a7VAhL!HBwZ;1aN{3Rn&E%!Fcf)|SXazI3+Y}q8>qNyXTTy1~8Q6`Itjk*>3M7hL zjpvIAGZE$X%O|REi$x9Eecs!v+=?#1GbQZKV0mvk1aD6N_#p_b%{aO^;Y3czq^!pi zGkuM9souKx@q~{W(H#kd(R7vwIjpigH$r(~O*}k=HbzJp%|ZBYf?1G=*_bHpmR;6I z$l@z+a8oY^u-fX^IpcX7cy>|cLK4K;bY|4nmcU!_7+)DHlxnvucHIo^>y9RLCv~57 z+ej8V!w$Moitoyd74@mR-(Maoq6B zZZ_zufYQAciS1mrE{UD@vW#<+l45`!mo|C=LxYERA21V%pe~R#Q`xm35EiGOI-;M86spHrX28GMM=6jAd#% z%AxAB!w?_W8xeY;M%d!}eFSWof?rr!sTSVQ#WD<{o`ZK zA^uWK@j^_5%u-D<#hH|3GSUP)F1hAf#x3^f_AvuXOY-40S5%AK!$Z87!WkBur?oDI z6D}?sLT5f=kozzm@Q9c{Crv}c!E%BcW#W6 z3!Ue32Ecg)V~{bx^hPO@x7uAp)YO`MHIujViyK4RN)@u{3c#4Z?t*@ir zA*0@~4ozsdy^Kxkm__%i^{!D?#VnAFvyt$5WGsCA7cCRkCAo0hX9*Nk(@eIZ`zzgYUB*L{E-t$eYy8T`t#$U&7^K zmj?s_EZthXWTC5z8dXLS+-pBZJAu9fe2fY+;3K3ksFWT-9|Q64>W~O6N=SJ*&%_^h zznVhcRC;t=@laG#Q?AnD;9_>J-btsi!E7@GMUDFkg6h9rm{+ixf2-8}cC0}xuTDT= zDNd@6CT7-x|H_M`UzQyW2Y{LXZ9CVg+l#W+58WSW9aB8SG~baeNttYxQ`GB*Uj-j2L#w#H`p2in37~@*9q;7W!v=*bcXB)x?1}?LYy)(42}Z91#jj ztju!$HP`@7;Ax@)*A3(=K*CA<8XdoRn7Q_bjn``XB{Ty;rrDG2QjUy71hW2v9EW4v z&JS4G3Gt_EuHWShir(q`HJ>sbAyaidfOWo(Lf~&nX7$21%}c@&mA36Ja9MZE2Y=d_ z9|{9M1du;~5Q5AKI|m0Xg1c8Dc1k@!-AvAL)fZD|d`9syY^IKA6-L>~T6Iv*-nHKJ z?Q2^JujXemwv+9HH*xx@u`r zW$ytRuWP9JII8BoAEXKd9aa5FQ~g7!pyen4anI{HcgNnanrRLzI(kB+(ol%t|xp{Lr3jw>n6Q^Hrnlr{=cGZKBoo-5Q&x->l zLe8TB?rrRbM+Pl8l0#J1nk6+xJ^Ne*RxU-CB5g9)j@gXe*8LSk#dp7}n)&7nJ0X&A zXe@#<@1kL{+1Q3+Rhifw6Q+D}2fd%MlvUeOPA2L`klBPdRMpolQ-m#?vxF>eIsKg0 zr^NgN0Mj=&p9AHk5dSgH*?eo+250Qr>A*sAb-%CnE@qKpL(1q|so;STYeOl!;{;Y) zRjTyT$1O_fM2s@a%Gi}W;UsZ#7%tigbv*V3xfciX)EVbEb=i0KbWD=uN(kfpr=W*L zi#*~!PN>3PjH|jmxfb4YD0bHgU*c=am2w|@wH2?%I}@@O%|Eyq3xP0M;VdmQ?nzau zelhHqDhr%E6%~n%C3#`lscoao{BX<2u0L#$#6?xOA#@DO(s?caXsUbPHxEV=2I|>3~@d|H<{Q~FG)|TBUZ=!thSbY zI^fB2mzF=E=GMG=HJm7fH$2^ml8D8=0Y1*08&WUODA&n?YL=B)wl$txdlRqnv!g?Y zHVOj(Mz$s4*OKQ4F!P_raX$qxqy|3zKQZsM5cf6O*KRDU`rhTU__9-bHykZ`oq@h= zAu9Dp(EkB-dwa2Hs&QUF3F_B3$Y>H?wSu)mi2hWM0gvDF6dbjO{S!RnCPf?2}o9c068q6>$&c$GkCbUxix;j01({YFcl4@7c>BPuv|l+ z+ab8qpS%IUpawA0?$AgJ```P37!SpDbLV|WaJT*&R=tkTMPl#JaXnMCYkCG(i0c4k z8Nf}ne2m${R{}Ykb(Bsv*fz=HE3XcM^ph~GWQ6#v3*Kp?vf ztC3Fj?qn~QaUHH^PhI6f8gFz*VkkKnS{aOj-G7fi(V$m9d5FR*K2kix+FcV;509zj z;wxAlvno}FvilPB^J&`WMN5DIX>(W0nFJW=4(LQ-e(GyZjtY3R5{dhdc7_~>sB#o| z&fymF;?kTQb!$s$#|O)gG$A+iwQP5)lX_@FXrqbdMJtWw9jLEI8x%?c-H=SkyESvd z<38xnaz6xy+f3>)oVV6v33WwUwkmXWBaE|qRJLdf7(zVF+mamfh8=Jx5d_{M&y zuq#oTKKz-F`lq6XEx(%*PuBc}Nk7UP(8;M!!cAy^Kefv5t?4vi4bsyLq7r|-D_5vj z@(ngRy49sInVx z&@GUASsQIahd|v7GVv<%XzO|8jsZ zcgfut{MqosY<{ZfqanyM&luWy#PP6jC1n(kTh_9Y6=Ul zuf|5vcL&5U;qSbpxA!{`zG{M4{n){z>r>y`$7A)_6o;8K5IXo$mtNZK{H_0;uY3kD z{L7br?qw`fUj7lp_GFR)nSiTKM;jdbk;7PkZqJ(O_X5e(={ z9!Q?1_I)T`0z`-_=i^oghU3 z@$XgzuI0ZImFM2@Jpn0M@G{J?2FdEs3JyWT6B9?!RhB0KObvDe;KZb`vx3Y!P4$8l zNrBO`V=mikxWF^C+-h8$Hb}Ysn!3GEcj4~4x7}X}?%~~Tv@hTL^fvVlU=3($egH~! znNKP~VzW%r^|%a82q9Ml`T3QBCOJAhNMFMFWB)Oi3Y@clF?Fnk_9_Cj?g3Q*@)gy?EnocOL&@F)K&*j?<@eL<@~X|f$qOp{0d4uNn6zoz69D0Qy6Fu{%@b! znv({l1MgJ~=r_|E&kb<;j>V}v4B`qMzyHAw1XwG82EGHnji4Kt9HR=b>GPB5A3@I3 zvRVHnLud}TOti0CTRwdXWb#)UOwRZzh$GPr-A<0{a4a0(2RY5>&yhb&hh(tq|W1Kbb0H!TY7kpTXCeekF)U=MnMGgrH$y^{j}^!N1&reE(0&<>9c1$mNxiu+U06Ddf^ zW12Urrw0i8H3nBJn2sFT+}s2lC)B~c=~l1~hrG!zJdwZ&sJ}->b(xTw!4B?GWZ> zY}O5c+yd@B8L+)0p3FN$|Ni>Z&{Kwh2*AP4{#B6y*A3EpK)4ney+MgW#)H_+Qn;^R znOE7Z-=ox01Ulxbjx*r5lIt0H?3XQ_Dy9xj-QNB^zh6q4qYbW^)>Js&$Re1e9${ia z!p(T-d+D|laDf3eZgHgC$==Dy?)NWBWB|ADf^zs-$>+0T-O9$#is<0XazIyRS(!Wd z$&cX6d&&Z0Eep0jmnL}C2n{U%{UI5MC-6a7%ZD2(#=8UFs=f+MpMV`9q5xz)hu14TAsmTjNZ6Q%fIj)IAMD+@WlGE_7p z48yl;0P&j+6?WVumDu!wr*_$F<3RTMdDc=Pw2SMS}u+YIPIXXI!!>-o3`zs&^% z@=;}PQ$jBO)_stBSAYNLKh)CwUQ7k8-&5%S<+pN!+}|^g|6C~hpE;rn`Tqtv@~2v1 z#%EsVG0BF@Tm1F--RZx?{x}gC%Ema}bK>g>*=(-PNOtdU9q2@hg@kY4Hm&umQDlmD zBC=@xd5w;H`L~j5P7Jn3uKsNL^B61ZK?_dK!mSoDMXuLR{H#{Vl$CN>!~&k|xh5|M zU%>Cv{JS06-%Y*#`GCI8tpCgheIEW1hkwKYux9?*4gc&02;?7S@Q*V1M;ZJt)rDjz z^nftLK}~?8!)&`Tu5?8#>d(`1=Cg~zZj1weUiGTev-5lYOG#Bv2HRhSjW7Ge%q>;( ztb@X+E|IeAWyDGsNYh6I_XO+QWae3&S1;4Eos|a7TG%Z^L z@n6v731?Gda;jL3M0OZW?BoeJ#Oup%$5Zi>wXWkYTAQrYPi=cVvKxt$H^F~(B3X}D zIhdGDpei=&NBpD+jRn7b%vBQtIJm*cZEG&ZL>690&iuGpq|7buFkKflA|_L0-Ia(F zbg~x+anK;+Ve%Var_6RI-{maqP;QT)r0bq<%ppuFWglz{@QC`6KId>8ro z?_Y7$ifxYEkhCG63G4*ad47cDbfJ~1gMrT&5>}E&E`3`c*+m&@>CP8~4Sw;PBXVD# z5e@1Xn(I5LtEa2GHcUtNAd_#>%$%H3IBt$oiYM}&c$nk-#=YE3&R!K+>=94$c+i4D zj*=p)#V}J^xqgH-9w9rJP1^J#lh9fYE$asm9-cnNncG{V@^U>cKa}P+GS5~=8b9<~ zZI9qV$=O$8B5S>l@Tn3V=7QKG$|v2`5$Wu4WyZm#T9`?h0l z(&PeNYIcyun#8Fzd{X7sh$G(D1-T-VmfLF4HtGUTyvKW@4)1R;YO6_F_{ zD~rD&-o{+OH6db8YPZOR*7bg4M8k4?b)Nqe5zb!3A8=30FLhBOH)jY!dsON7PQhU- z4-}1Xn+tJnS+vlaXL;&VD!JLwczp)$JJnXLU+Rb$uPX4Gsp98>t{24Dd5o1HDadLowd`w4 zlwc$7WO0umo{_SfGL_co{LuMvLIXA4xy?w}r#;xB-Md7Ie*eXzzC!9ZMMd411p?92 z+N$jABm6f-#jrQk`<;e{L{OR2*_?8l19luv=$d|bUS3J**V3)(2Z9=1wet-aqYX-T z9h6YWE%?i3q`3*P+xv!zr+PMWL4)fXw7erpme|bcw7RLKq@)CD4(#k!e!OAASxeLJ z+>;0%JT6{JBjIw}!PV(IufKg;V~Yc4vx*nzzBwcAuy)Fku%?SFD>-rJep3KUYQc>v zW3jxip-rVsRJPI+oiAT8PxO%erFEidFsj@Szqv6Z=hggK$Ta!#X`LJREQ@d{^W|GJ zEn)fNndXlX(anSbnD5i^WJfX{J})yqq<7-ZubS;<4OqrJ6OX*Zwvg#}u*ohjudKuf zo7ENP0>bXKE5nH`^ua1bU@Ikl^`h{)nQ2si6VeI|j3SqH3XXNIDr#@-B>Y14YMr<< zSK{F}Ki${tUNG$BjQw(Y!DsPjLqXl_N6wSw;C3?9xLmZ%((9Hec?*u9+BKsIF4-!Q zP5$Hco*XhuHj4UwDYI@fDOb)FAI8KPns!-ij5;RZ2P5$buM7(S4iP9?#>_FSW7vjalH(>mMXNhGvQvL&yn%7z-M%K!_B{_AYAbITDeC!Ps z<#%3pV?7B27rLp7cZA_fyG2VH@)b4)EFOC;2{nDuox!(WGrBQ*cm}802?~zz|H*yR zvzy~;i22ZAtB#>o;-7pLrY_CLkbW+XcA6&~n9#}P_;+lKJNDVU$~7CK>3-BE$<8~P zCt$JAZd5UjvnUq|C_kW~ex8spIDB_D+Q<=tGz=qD0Qhp#7J*g>}3q zLtEU!ImN_Yh?kZ=nVAoN($vF^NYezew9}7rOPm3T1zE2>ob>57Xp1U^eXAsGs%`Bx zS%;U-XER!u5Qop)TYRbfAa`WoG_%PmQ3upe_!DxvYVz{4`E*sDwX^75k*qOf+3ZWN z#EIuQIoVg%-059xAlo@WFNfUf3y(!@AaGa&bzn@*q}#TJs1I9|Ag8Xb$DB1C#O9&7 zB*;UWTn%r^Cn&RB84gCy#6&x(W>M|syE{IqVdd0G{amSLC1zQB#qMPK+(ATq|587p z6d_NxB$lwmV?O)NDa9wtT5i?kKARXDmE(5VJl)4A3svbOBTdGaniFgMT_$a;21~8N z%7PHMMCq04xMBx<1j1EYIsWpn)BIt!WcN~+xtW!+7u0SWzis7Y$~;#XmwcGSlllI- zU1j>}Px2@ylVGd2lqoBA{N)IlUuBya8fNY@7s4E1)u=L?dKMw#s+zzEOxJyJ*h*t* zxi)x^pL6VqF-7W8{L*z(Z#_aY6A>5E{nOq)YWj0G{$;6+lLDGJbF9?#E zJnP#S<$TIo<~U&|6yZ+J$H|Yc=ZT`?qPr_MHkpL0%>5Sl6WbzWJoI9!_uL?kca+-N zm*bkQgz54voX->1wMNq3Tuq{DlZh434tkkAv_K1$knei5Bi>Ax_Np{HwTE3f5s&8|# zMuVfB$qgzJ*+IpM@>xcu@9>B}durJ<`I0YR4Vz1?vfkJXldjRzl$!qpN_jS2hXv`2 z_Q&=w3C(Rzx8*xA1=L1gRB|2LU39(Xko48~zg9l7 z0Do$eN3OR=??nE}$uVsE9S>R8-tR7%#-2;k5uWaGL${3D1KC3t@*j=PHTgBL#Jf#F zyJAsmd&Yh8O$xj@*ys!MqrbTTX_wpVy7B!;7W@=@G0eN=f?e`X=~wf|x@#_0b@)zr zUcOJbWtz#{)B+66sZ3gw)~Bv^G1w*5X=xgIO@i;BNArkq zo(^OP$#dym&+^{f$V_sXw`mFI-1HdU>u)I2O>j;CX5ne_6|35XEx30}NYH|?pLr(= z8`kvx$%UM|8hN=~qMrKHJ~J;=rr&dvBODmc{$e50XfDNJ;0GgzY?1d`RiGd_NT)8# zd$it-Etk=)SO*^vX-X}KG&L(Jb$qljWYMmlXISf~WZ`OOM6Fl4deu~?mKtzhxIjpF zY|UX$<6u@sT+&SL}VLhyx%F#>@Ey~ZkQC%Y=qZ71VeZCB!a1U_1 zpwoY>_RqFP1|8)bOg92O6tFT5^tcF6bJf4AgLb#sndN|VF3TfL=Ku}GOR(-WuYddZ zS>1o1wEWKpv>638t-p^|{_p35f~X}j=jBL|Ab$IwvOasHdHdF_ry*(QUy*LV)LO|& zpB`s0%6G`m4dE;)K-msR<&+NI2`w`ePt(n^zxCHSB}+`L;z0~f`QSO;hKo49sV)4< zXw4Q{ayD+oYk`QC1S3%R@;9a7|AIHmjn(vB0(m&V2bxZP{W>>44;o!c2fTm(o)yEb z>y)&|h4Vk2;4$t6I^La@c3%apLO?-pUxmwzCoapI?Y9;9@SEGBkX_<1A~tT4#ru06 z9$p1(vM>)0+e5z@GOSnrc?0=p62M;hziAS(r8dwI-7!D~Oa_f0J#lpG+W+bkKn(?@ z-`5uI2e5s5{PR8U26_+G)@^!s`UYtIX51Bdh<~>Lm=m5CJ+}a@4>VtQy1Nm8=1 zFd;HB5_BeO>4G$>YH4XHDk_>vf)8ygKvH)9&cK-UdWiQ+>%z{=sa z+@B5Dd8U#lLqWTtrgC|RizcYGk#ckI$Z{Gf6AVwx-gn%*=ur((LH8!GcVZ5x#%hiY zP0)+EfknP%-MRi1gMfa)I|~?O8WiFQy?L?*s$oEn*HaS|NaG!FsWYOspY+GU_Ff$> zo*Cisudp7-(e?KBHU)iDiJ`)EZ=JwT(`@=Oq%~pzyUq9Nq=pKWbYfbMyOg~w;TURzy~FC08u43lt;GXMg;=0HzZGq(ZVjWrep z<39llkhHtuH3#~Cf{xZM^bch~IVYgG{}jl~%*2%5IQZQZJ?1gtyOQDb#<63X5yBugO>GG!I7pC;u%ARX?`)|76|Gzg^c~6Em zRkJ)SwV$Z-1%2V)1UaBeY(V4KvtS{G`!du<${dVLC1qu6frg|%DnQJkpu);)4m5Cf zU+m$fO92Cr7^AXl@PyBwJ$I*qT4+mN^n9)jkXcPhn9>VYU%?q-ipWpLVQ51DNl=8rQm@}rE>OD^4 z180$-s$@k<{a%;&GJ_{n+tM-9fMnf17zBX~BziTq@h7ANhU6!2U=}Dl&7Tc&zqwOf z6rXQ}`}+B`e&zSDes~Ku^+EKHhu)Q(3oyC)NUGXr87~6TuGk$and-NaelWpj@U;A5 zPgX*CRtF14f*7H#qKoz4;-U7X=*`(Nr{tv})J~t>uiI6@=;za&InurzebS0Ti47fz zs=e}B>0I4&F+rj`8Z`e}=t`t9eZ83U99uh~Q$OvfUPcpNw31tD%yk0T`fnQ{&WPCW zhV~(w+@XAB4wlVXnND%nH+oaWlLv*qqHcmg+R4}9Vj_IXgsi$n&Ew}aJjq4D=>TY4 zi5EgA={2-SL4Juy{9*g{%a>k_seL+!i_L1M>S5g`W!ax2etdmuDx&z~ux-N7CXVOS z4sK~>EBv^Y5!Wx`&(}DI3g3p0l5A8Qep;}IwG19hh^gjr-0V^Fjf=8S5&9MgX}jn; z@GUViIN-}&VyR8!I5>m8F+nZ&WLIppwGwF05HLiDg&#$T62pmps8Z8}sv{30Po3Q_ z+U(b|Ep=|(7jiis8@ix-`}z}85koMHICdxURCnYiwU85Z!_P;^7W>w|lAlx+PR4da z(SA<|&8$CA-P~T=WFZgc2f3R^_xT973QFOYkqEQobg44i&lNK*BjO$8-MN-zKRfh1 zpxK+Q&tHIH@I&tl<|vNFwi7GV7n&{RSNpRUh^fxa zYlYR#7ajJ>4JiR_CEdI@MJ4{2=Ho58VwS$K^fG`mU3aBpf@cbV=Q zSYi9*$O0a3&584o!TVbsVAp47eYaSiXR5^+QmoAO?bsd+Qoxo``dBW_O+Bb z_>dM1bSruNRkU(_BQ7w>xjlnFD!NpVxGxJ0mS@=kzsT}zcA`nkn|aYe{`zA$5L;%j zKhBM0YQ)A-W8(Ttp)s;ThI$J<4J@53nm0gmd(Gb7co+LaDVsNK`wLS(Cq{8~CqZ;P z=;L+n&l<*;e+g{a!Du#$d+xK9+X~q@mymMMfUqVWi*I@@_2Tj~rKm6AT&Q`Davy1! z<$wdY5pW&u;%wt5l5N>;E$>;tqDnHN%R(0#51{gU4qhJ|{JH z;^zhvrJ|wItM5fE8cmRLaU0kW!4YuUL8>}=p3!~r`QkC(gFDjZJZ6Xk)9~X?+O(TQ zORv3-IP$D!XGJ{4LWCF{^`@TpVdrQ4cT5`U4_)(2>uMh3OP_Vda~O%HdWF71kOkXg zobqyx?$e1rnp`I*MXe7Gcum+w!AVvf1~+|sM@cNnyaA88GT2=&g>mC&6ZK8~*4TIi zb5h$a_7B|Sqf4RfHM?HMW8X?E;x@pVJkR4HE?8Z%Zxs+1UHhp6Ol(#KL-ik{JoWOBD$lbfu##O|yD z9@hK%?}yD>{lTcTaxr>6+1=dSd}^dD3hUMT^UVG%=M{c=FXWBB%{uOw!tU=Y1#)GY zUg|NPLmJ7ucJ0yG)jm0SBPqlC663jgDv>N2iKHJGZqKz`K~M(ig)&Pf&km)wh+fu! zLM%_TIZQLP@x8uo6iSUeiLj;+x8lB<3s#el3L3dCr)^@ zoePbQ(tVWbs4|v=VpU*hOOW0ykTy+>$DOmT?TQ)4kGv!f6p)^+xFL2)=||9U6Y6m_ zRh#jDTn;@_LEk)DtGPI3j;9A`o9ATx&%c17(2e$Ls$d1;+n;Z=t6b%!eb~qjxGLFi z%XLcwAy-uf#hpfdiCd?=+9ki9^+9u$C;B~ne%nEOak$#9-fFDUb-wYu?a7pqHsAQ` zKy1n%_9Tic7yAy8UV)Ua!E|zZ<18eQ(-mbW8 z_ajR~e3qRJg~cGHeq9Sp$$g%Vjv0LgjPd;i=YHw4ot(kAQW@gE`|1x6r z4QBktHaPi#N0tpIzcgPhw4^0JEqCz}yHIv)Vx)<&nt#RR1-cOUiG^{A%3)Gb535zF zGku|OlX|c!SAXB;)(-H4AAk5*S^R7cU1>g;7i+e_u~O#Pm$nRchn!@nGTCvFO?$8j z6yomG9dJA=Rsl`_2|Z?1?Co%?66BXZl(Rg1^_*@Um!EaQl+{|3*HxsRe|HJH;4u1o z-hfwo=F3j%SoNGd2k08MaARk;B~f~2Ye~|pX%?;QeudZ;M{}f@E8FkN|F!)q*vL#fHd`0$tHF`*)Md;Je$Y zM%;LNw`&2c$K$Y+fkHE^9YPMJmHtc%O#*Qk+gJGcSE2C0-rLK&+E0 zI%=NxQ8KqxhRN5AGHW+8aUAne;GCAV(1@`@~55 zrr)Q2hn|#zJ5Axks&%_1-`~yMkvG=m1+pE8)#m0Qd~a#rM4Mh=OvdK6v#1(Y=Sbv= zKVx@EI_?xGAYZi*QZFt~SYEUV6R%eg}9Lxo34u) z^=)!3#;s46WF9oz>F<)iCpi^<46$5iB)b)I47fHk{iuZ?@zE+YdP*2pDR)xQzWv92 zjBvts;#Hd-}o3OES6>tHq8woJW5|lV^k6!WTO54+f z-!nwgk3=8N9<%(uu@u$+4v3-&5XESqrY@H@Na9G@TPxji)cK*|To04niZ3T3?a8%v=dugpEmeE$ds;ZTfPB z7;Q~xCNBs-{rFl>gFOH4r6=c#O%DB zW0;Bk`2mP!+7o$$*ekj#8R5CK8D%GK-`WvAkZlC_BR2~&TO%poPiB7|dvYvgvvG>H z(N>#ik_?u8?q2|14m)PA$PWVL_*ba-f6$wUmn30S7sXTyYT@N^a`5=ORIVeieOF>g z{C-@qQY9u&4*SZVhd@~N-M@WHM>);}??RFSSc~P~wRGqIC*SDukJH*l?CdXc$VDVZ zeGswh`oX?3=rnpp)e(53_qX)8!dJb9_8qm=zkMK{*<>k?F(g}y+GjcFuN*bWD>j0L+(Th@dZY;v(Tp;Oa*xnlfZ#D;O-!^a|)hAZzp_mXdDsV1E(PU2f<*2;``&9b`P~w#<&G+gL8H%81&1j01C6#Z0{lPw?4@ zkA=xf2D~q}-cBzc+v!oY;DBYm{G{hARJ(?2fJKaZ7l{_$ONx2^)=i=~V1(|pJY=n9@|C5G_SU~8g^ zI{!q2VmF#5SY7up_f|C}X51O+pnrDkX6j=3$?P;?fT?_nxdjThA6^lwaW}}_-XRNf z5r*U`)j@|hSQXLD3^S%R4iC#;dz<2R(ni0P9W$b!-D;(9)jO4AA3>wE`818>Q==0IPXC!WZ-MRuGSCXHahYP?&?6EYz{Qf$=ERq5SFj zfcK^5C8{+34+e7_7f5~vhqRPGU+vxKTp^JlU4!P z&<|UgVe=+ehXr!&Q( z0j$rT$$?&M?Pf`l2BVy1>agv8ZJszb0XJduP*W*tK)YFNxzSKjXdd2)`fM^7_h3|q z)cUQ?d%lBB$k*lB#z2`ckK2}iBqmwYC-Tig-Zm3hc}sc`{Z(znF>)1kag!56+)|~U zzTq}l!=S1M&P=df`XSjT=QV>T!d#Zs?G8DZi9P=*$!cp!?cYOYu6d>FN3lF`!9Y{(OtE*s4={X17_L z$PHKWH>V5$pGbE}%Qz?xV5!w}v+ZWW^F{CSE#UHAKaJYm|G0oxP1miFCaS2ShDL4;MXWEDF^T9k9++; z?Opp@lGzrXIrD=#p1h2=DR1L+jLei#D>XFInxQnoG&Lh7bxfmCN>dV1rqXh9vZg{$ zS(>S7ns?0-lrc50c|#E)z|h1C;FSavI2&f3=bS&{?4Q0LzUSHBUTeQ=ukTrFzwh90 z#2yipFmt;L%8bp+9J2~zuNU8pHgb0yT6>HhR1l>Ftt`v!5^8b>d{Nq#_8U(YoVq&V zmg?~Bo7a=8Cra5)t>g zc%4a)+Yrjzvr{iJK5w>n=;pR8F(LsQaqJzjB)|%$>c9vehXec3t*t8M3A|^`Xl*s< zd;9K#rl>+q`;?-WLy}`t%ILDD#Z*_K8V7!YKUbA2Oqs9F9p^B@`O_~bCn!@mE-`Xe z+;e10;XvzGRzv|g3vH@AJIBMN{i~?#{!1x)2yU{|uyDc_)uOs=pnh*SJXo=+_=qr- z`2X=gNcnC6+qlh%*nL%|1=`jKc;7kh_%yXE-+f0*=8^Bf0!3oD8F8%&Y8fv|z*|0P zE~W1~AQ1=_uWb2QEL|?)m{KH@a1Mv^F&~3V>TTC6;JjS=Vd^0S&PnYlb{jk>JCW4p zIj`MrMf+LhK|13zM`$m=xi`Fp52W(`3lqLWpPL6Vs*-Yw++y{Dxi0^+eR}WG&TM)b% zl{$qTZp(lTk$lpoct&+}?M}I=z~+2JXOxSF zbd!d-vk@`CCok;(h5LwX#t!G-uAG|SELI}1*dF#_upwGdZk5<$A2q!4!Im+PLKHIEU<7Ys3LHRb6x-e@j&I`tTi>~?J&#QDnCF>3B zL}uR9XH-=_XPZ4BU~cep;#g5VrTztuuaZRaZ_0%~s=s6YuN?lgdu(+3j zvC6{KRp3KEqNzaG&#m&pc#utNIGtHb-(7#095a!wt>Y7bdzY>aWBAOoiV(dH)k*OA zGFFDxeaP|ttu}1;@VB`(Cnxe8K+-UlrEy~n5X9WI2b9QiPm$(y23vCjA}V)d`I!d1 zoHG|z?5YRs-D9}*0#8cond<04hYYO?7&FK6X-rr1cvzZCMf!o1F@T1rr!`b6#<2o> zBfz{F@b=x7Jot9yBxDS4U&t)(vXtY_EhjP*8*Q|HTX&Y~LqZCv=_iV(!2HDPZ5`#g zR>ZVp%hCM`{vSMAQ)O|hC{W|TzYVuv#5RqKySp*GO2YUZ=a6x#mAWok_R|2eAP-X2 ztf==*NlSgtrpd~!>CVUDj+BF=6L;@sUSLml1ycfiF5=$>rR^6%_c;%qO^*FITP z+9LD*k?RyW+pDQ-gRn<#!EOh zIPjI&jP|C6kX3KB(SH8+`o;1{F+o26Dx+Yyu9Z07?{Kpqalv^JiLSvWB#B>b?GyM) z$9K>ynUYLvRhYR>VNmH+g^#>MYs$hv*(}1@~EqSk{8|m ze&(cqHIOVOV~}%_h;&7Tt5&F_sqF;{n%a7ucRELPf&IWSo{2)}+Lgy$=dM%Z1@GuQvq0kd=X?-u4oUbu>Ri!DxO!i} zO%`^=e&bUXL}Qax(&>zP^v(x<)$`s=)wNs%+_@~)zqaw7uIuvEb#6-uLcvGnT<$jB zfqvLA8zfqNs+?4d+w&?`FoT#z4 zF7c!*SV``}SxbvNV^xi3;sfw^lU{2W-5=V3O^AzU9kfR`dQee|y5}hflM(~=nn;S; zNo>5>Qx}N#;xwe{37IY9j-ewy66%rRzpNLi){9b)@1sJ%IRVCZ{(tkl{Qp5?4VB*b X(dU_8=Fw)=);abg@<^4}*=zp+P^b&X diff --git a/figures/example/hp_performance_rf.png b/figures/example/hp_performance_rf.png deleted file mode 100644 index b869b42de284b350dcd1c95a1a5b7e610f9c91bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48456 zcmeFacT`hr*Do5keJh}%vZd-)DI%btbcl+GfPge<0xHs^_Y#t@MMXeCsY(~5OYbF# z(jvY05~+buLQNnc$$bL*eb4#sIOF_r&N%1$?j1X0pa^TNr_4Ej^Eb=040@=k!ty)k z?+^%tMfLt&9SG#e4d&O8-@s4mBrhg{zm7WJH*$kOL@zSG_Iam4Jt2_u5Y@Z4^}OR3 z$9(+yW)kLB1uoq>@?=o?Y~kDUe}-5JRrInfND}moO`Q5f%MxT0x`$nsGD~GQpK+9i zlp2@2JpN^z8^Dp%aE3$g>Xo;DzHC1A>csiCM=xAxqD>um8-D)Rmcn-*41oE*O`hq5k%dY_hqLPMl85x3=TVOZ^i* zN&g5nrcP_o=i^n0y_PWE-6t$Qt3T@5S!*JSnxC@#6GKZxRhQ=E3&L>gwY$H!2?e(a zRWi_cu8bhk#YP+gAC{^Qhi-vp8e-Qs9&~N3(;rHgtDDwDF?A{(%b26W2 zVluL287JvbbaU@1o;Uv`t>!m_H&UzINfKHZX*}vo)OLdJvc%%g2y|zTWf^`~q~Dj% z=cCbzBW~yI2#mGTo%{#GTO0-WutXzI+83d;&2?h+mdO-jYq5x+aWM?-(qK`?zrnd4 z7w%BKF2pEZ{vmOUffd-@yzh?Rn}gz`4Gagbq;0a~U%7HcM=k4l_mH{#6ay`0ysJ_x zLk0M|j?|y``lHUA5z4J`+QA6!&D!tHxA>n9uV26CMd=wBRXX->8ps!9fBpf3ugXD( zA12C}t*?!F$s3f->)|Z0NhL_L)WU$nl%8GnOSYv4#E<;`J167dHxs!THYpU?dFB0$ z3CLA8jOSt}8g>>x@J+eGbXo2A6`^Z**trX4q@(-r_%& zDE-4~+hL@voy7g`E9@UVeE4BWXzvkJ_WvwXUL!0l6e}FwoyPeyu|V${d!s?I^>HpGX+D||5lxuzJUR280&%0(k_$dtm6ykEmc)j z!*~^nr1^Fy>>7cU-P_&&dm#Luv{V139n(L1^1qHXgZzDceM5=0j(7H$hQC7*f=02| zdMc&;VQJ6neQ$RhF0lodMMXs^D=SkMi?uKaT9HDkY24qGBk9*hkCr*w&u4ZkB}6p# z+S5Cp_~M*(!@X(!pJyijD8TnuWqpd;LDIBe*&WU9GhD{?n^|GtwL2oxrLb2^Mjmx|DJyQ zCs5IU1pYr2IM|;5%s}BE>HH&||7%g{Kg#yMrEIrFvokY){vGGI#z=#I=j8k=?r}A^ zT_ynelK$YcxX;>jY;0__h)J2_7{v=iH7&E=jb}f&)gN_jY5gqWgx;QQbr~shXxQ}% z{@%O`dpV4MA`i&_N_db7awzA8v(@qPxxCELc3=dGBW00NPFKN`Yrhu1)_R>>uaWI1k_`Pf?e z%!hC1%eye(7cq$Ot)4??h=#{9O%&|9Jq9#Yus?PxH9xEN zZ+`U?_?WoQim2!_QYuA< zPf+x#H@eK}PdnS73MsS<%Aws!o~OFn&fFcs5fNWY6w@Mu-z32{Ukf-~VHZj}y9D*> z-OA~Z`zoCl*`19g#um9(FPqBH@?cg5>5)2JokXpzAp^;%ph%sT_QQ#uO_muXHKDYU zr|r{$_yH$b1IGAtb&^R0$R#Bc&p{d6Msdw|)r9ir+*D`_$j3Lz?7-E*9LrTs%Wsxq z&+N`;@s+Ef=GA!0N<52DTI7J&C^>Y#zdr9&!{Ma$zS|<;QI6x0n6{Bnm>Aq1lF>uou4Yi>ro<`qN! zU$>xK_1MjH`I|m&uMcWADn9!>)R))$3DzpGRBq)vTwF!n9A4_qMMUj1aBzT)eIenu z(HT7=k#k14g)@}0G2gS8x${@Rq2~x~JM*MsXV@~^?v&+{;)B@L&x$oIZanNPCytw( z8-U}`%AZpkLSpPka_#VKc5}S@5Ak%i5(2HhnUv>EWNK)~%?6ed`bCP}{}eIT(J3}f z+Ir3XM&c7+E0_v}R%mr$6{|F!#ziu7i7!m`)g0oe zZ(l~U91|y~N7p2E_Y;_jb_ir~+V!$#WWH#})%g27a-PG4pPC7>mXmjlOMUg#lDGv3 zG0|7sxIN}a(|lJWkX5sd*t7Z$ikS3ot{Ov=Qi;f|zi3(tjBqFJ-a_re!$z_jboCkZ6gL)-FX@h}w)*Z9Y;db1Zei5VEs3M_d;+ z&zxb2(<{$IEWK%=Z%t?Sroi*}2eJ@)N*xgT+uh~XD3a$8C6X}E4*%60GGOX5R;d&s zX*S}+_{+4%L|8{H$J`sSEVG~N>VoGsvAv-&kvuLP3h&|)M=kDn&{hUWE z+|mW<=~nce?UA*;?BaLp3kH+*6b(j?Q{RgPlSk$DrPs~$^m9iDKi}8&JJf8#*v2Kx zvo&xL2=5<8fuyUWHol<=NSPZ^f7Gbtqr7V^(hfE>uuv__hf4}}ldup^jmjd|l!AhZ zBx;r~+(XU8NM5_x_UY}XakaC74*fPl_r{ZZ+Bv(>S+_-~$eDLtx_LebQ#FtEbW90l z8c7&*PH!1p#kWfntLE(px~4R*AueXLX#Yj_o^oZzwo@y@kJ1>!rW zvx3+K#@#v;M%E;DvyR`b1q>$d=9no|rL(uBoT8c;B)?N{kx zEem}))@sLeU)0dPt(2hfLsCqyxc&Z8`s2uM+!D7j@3HSVh+!pc#fHdAV$8* zd{)Q}VUv5|-SVDTo{59RIMY%Ei)Zn{jw&LeBKD}#ai=py&#hl-ApRQF`t zE)+O<_l4KWXY*wPpig~HqpL&%+Du_~_=$WumAd>1myDaA+#ld~Igj4y)%RBVxPPw`gCLZ(%5jmD;&6_VK>Hp(!7pgIf+(|d;IWGl+Qaovp{V;eWXbN z>iQI$iuUSfhgQyPW=AUe!MkS|JNJqEs3gK;y=trZol4@j7x&LN1R}^qPHbiNd(M08 zwq`?6Wp^BY~b{AJ6>GbC33mBgV6W zPG)O0GmRr$Q5*lHz4kH8v6|=tt(b-ddQ!JgCL6Nb83x~~t!h#EQ3GjU=v6KWXpwko zCUF(VD0kxPg270gyEM%|!5|cjuCrjKLT|}`^BztQ5=ci$Q&BTHkMeVEhtL-IW%%JR z`ec{N`(=Zgsgudd11GI%*!L=}-fKa&9UXo!+abcusLk9F4zGEICqGw|(C&+$6dmG% zc$ZkW2>M9b>9njcMFk7e$g!AQiZlGmXsT|E0+m~Ob|p28yC*;3+MY5kUnEZg{Gah! zL=INa1K*D%+Obyt`q9MUw^mS-GM=ir)0oMQDMYzlA>1THy4_1CLdERena3X^S-(d* zJ%9UGne)F=QF>P;|JN%=uHQ)}|LO$*kNPiFZ~xz1+W(iAvx6ip%gbj|9{<`>tquSA z=TnuM;LQ7`zh6-k6Me`cXx1HkM(V&BlPW`}LJyR0j^{}ir~KU4nH7-!+$%xJABrkW z?v@0boOzXU=gw~j{yI=J3Vr$ZYSRgR%(?Ime3)XyX9;`_$>%_XQ@h;mF5muFd&R#S zFaFg{ao}Hl7XMtr+^G})>1upG(r1;#+cxc5O!&;{pa`6M&Cmf)>|c1xLxW`s&E&2E)9!l z&Cbqx5_)Z?e|}VtzN-4m9>#3NVSdW=A3gH?agZY(L769_smrAN=$IJj&PtsEj1(_! zskSx5>CYdu*B1>5Z%NmPdvp9;cv6!2`qQ+uG<*^g;p4OY^W(+3X|Ui$ zUv_tQ_uY;b(Gz<`?IV{+;}7-pN?fKI?{-ultMXimojXao^!Gg;dd9|}C`us2Kcqd0 z6DvnN)X@o_^RWw$j*kBPxsp5%6EiCMT2td^fTJf6{W28OUu6EhNc7pMHWsjbV9HyQ zA!4T#oJ6{NdL%4*K$CAtNr_IQJtM%lop=u98%v~h0QO{Zt=813nftL z?&;}Sa1IjAuUVT2x+5{h)N=@QR&h?Po;!CAxmCwU@c8;hWpAYWHO3Z7o1AF`Y)gz-es;Fn z#?PBsb>eAX#X)-d^)U|3ILX}pi@LhH#A+hi4CQ<9Q}%_Ed)r1(zW6vj^5e&kiHYbA zMJgD1EGjyh)T^sy>e_S}?28jiV5#Gn@7i>e(wpN)B@F|#lU!1=v#(sfe0jD`7;RW` zwU|ox7=r9(eoow3M13hJaE?(dIbZYKBt6nT_NHBQM1+L>V8KFOwc~h=&qz&dVxr=+ z`nOz;6<_v6etMOeBwaXPEHi~zt zyTWp_{!Q+Hpiz3S4GZTDo6h9hK#7{PRe1DubaZIbKmvTeb6-I70l#ONg7MDO-FXZz zwXYINU*GA|?jk0=b@)>RgMUv%tTjNbH}g@NdY)X)zQmowIAfH|!_ys=uD?DzFg|*@NG0r+2RY+; z8s97p2;Qu*R0DN2`|hub(4&8lPOMS~lGh@jp*8NlT?1MrvXeVpPN=oPKbGU(rnI7C z9{5eYTSvf`w=M8z^%&Q6rI)%-dhtr(V-#`~m~|Wf>4E#vLJp{VP2!g`iz^y}Zfemx z^3AtgTWr}lSHHW-)Uny4k6GVe5MLp0=Woh8eSaM$misP6aCoJgv9mE=Y|W^!tFhW` z=fgjNmHEI8-qr2Y2^3k<9?|R8nM|8>X$;VkNu%C$ z5(@_Rt&&EBQ}!iHx;$cz`!?iI%>!HJZg?z@%G@k%7arRu8KM=xwX@#F7R!s+y5J&X z;qIMpX4{n#43^&z!lCkU7i4asiXI$2|7oi}NDMQ&ldKWQ>QbeVV@oCFrrPMAR_+4U zI_b9SmpDJNa7oq|-OHK~(Io(@>PgpNR}j0UF&S~HY&7O97P<-Stt?(4%L&HRn)8u| zDj#Ic-c*Yg+Q*{fH+k3B_jGO;n0ckEHHh6|#4(he%ixlX5}hvp`mDu+6@;q~qhX;M*wz2 zJ)4u1K$$dX@?+GoIhDp`FjxF5?yBx@=l=pq6Dl*YwEIiGPBT8f>kPt0AAgum$^0z?f324caSG&DX0E zzfGn>Ln{{on%m5m6s9-!t)4pt?tId5O5I*iL?l;nF+>b~f1{@KElSFr&?_V&BGR?~ z+rzNQ4L8OkC6%Ne1Yy50SU^gHR{|V?GvHOALJ37-PCm(l8 zr~PGvM;gCIDmCuRuLXwkRTogi2H*23x{qr!K1n*(ill1IovOp>rZE!b=&?y=65BKu zbLhb2R~IBY*{$XfQ{-<|+X|nyaHhTZZ%(f8!BPK9r?FvfHvQyA6(`?9T$EUpJ>nVd zaD=GBY8P(voTTrDOh)1$aZYL2fh=>{QWzRf?v9@in|_zefvg|;Mcrbc;Vv3R7wGyG$a7_`8#X1Gh$^`X4_}q6O;4k3lhP~~j0JYsh z-e(@I`mSl1egFc?t7r@o3?7b`c3ci&xcp!fWsIaa91kvgFVV zzExf(8s4{(3t<`grVVTv!s+iT2oZrXcAAg~uVxmDylu=*TD3dJ^A}dc^*lzMNKE{o z@+WYU|Doj7tGdI$qK#ujO?7naUx6ca*%`lgP8O^?CTs_oqe=BBcHSMkt|ojnxZ9kkQneKjCo9I=UP9z zvtFkaEoy>xZL8{jA7wi)Vq7*{4IfxmV4e3F)P{#G_BXeMfaiTUGJ<`>lc$|s_N*be zxw-jn*y9Lb*w|2>xEqDl`yi(-*5iI;iIA6WKEavo|LrlerNriROJtc*>hsU1Ad5yj zFTqbBi~IiHNDT4c1-zpL9`>0}cg_1&z%+(QQqC(f1uT1?iT?I{T%Hw7^oJxn?$W(M zp_WJpgrMX%S-;<+u|~h#EB%J5Q&+afNc$MZqy8gyL*^^IxT2Mb0CIXfTrW&hdsm`` zJmxyT*!H9cC2~NX6a#dvf$mH+jX~94ECgu&;oBvax3fJTSmr6gY(*-x&1{K$VVJ6y z>xOU7%E}7jJqJk^4Y$qJwQRz=@zr=PD|wNq4d%Su)PR6)c7+q*o}e!27@<;!5lMwH zaxLUm>W2p|&x*67@<%;puQMtxh!H7M;_M2{qkgL=fdvW*W@csrh`0&t1M)-)_#|#B zjTkeqlyJuKa!zn?G?FqQ@Y>P@ilIbt9YQ)02dv+3M>F1lRo%bG;`CW3*DeyE8{};a zSc`BH!dI%}q4Cxa;SR+OtO-MY6LnL+Pb(KbAN*8`gk11a;hz{|HkYty{|8J5t^jhW z`1$q^q;FuNfT?$n>P6)B^(u*ub5tFd(*P{dD`nTokLAhT`%lSp18d-I0fK;pKP;*b zYSWQpsS!e2H6F#%e9ZtvY7ZUr8do5OkA%5ku7X_RY31v?C=R@o-)aK~@F8al$!|bs z5n$?scfoz@sd=zPatGg9PK?w&?b9NNtsfkYa#pw0QZhgQlqtrXW#{0f zr}1x7A|0V9$NIJaqA;K{SK_Z*+`jQE^FSET`vPH3ym%UEX++Ad|0+NMNWtOz&C(~8 z&jq}p4UCf&3iyLl=FHdeF?Ur;H8<{q`B^-&v(Ge3)uVZDug!q75<$F`{LG;~7^-p# zr1VZ{&c&>K-^d0q9i~VzIL(tV7sRyD^#1_HkQ<&&;Oc1V@(6$MBjDBmlnD580%s6* z;$J|k^|zS)4_&pLx*G|C85vE`3ntoC2PAS-TgiUVcUS!(1=gzL(yJcfHh_fR5dhFX zDfk~q?z12k5LxsEddznNZhyJ!;(o|2o%_Sm+S-O&ec~CC4#St+!5PP7E9NpF)t79i z>R%3+<4U$C{yNP2sZ$&Ls&MG3{UGAKPMt}5Z=nU2Un87UeF+$>bod6eI%m8M=Gioh zO@M6@K6O3?m+!NY2bTOzJKGAFGUU>i>38gQNxU%f;W)nXKcrWn?6N?kFJ(0XW&zHb zC`PkFASCyuKLnx`T)VU~PCL+3Xay4)gOk4lizv{r9~=U{LHN!23$`cENT;(d@?ZU3fvpfV=5y8Y<+>WCADCx+djs|G)ZwHo(}zX2{Y ztw*>ic|G z#x-zzG6;J|D_qJ)UDZ`poBBIqA|oFowr2*+U`riH`Z`Dm6e@?D4T`BS@-sMnNr?LaQsqN zQ)Z^mz}#W1%7fOSU(M7FaJF`Sex8ZFV3BI+rGUS(46sOzo>2dsNg9U{X^Ym-xEr=Q z3M_z->|tYO#`VcsCfN8ZFp2R)q9Qn|CP%Dsiy0TUY>vEc(YP_s-m}~af_Gcn`E30h z!Uo&P`@ofW?Aem_kf-0jdjVi2U&Gj=>=Zw(YWC=cL9cZMKp?%$c0ef*^{P8%WoK7S zh05ijNW;c3C2j4l#`jzcwrdpyOsV)gTylyRG$l2&lkpEKh=m+6tcV(B(*)^>kB^Us z_rS$4`L&-H!|LIm#$tAJ6+=M#L>SeZ0ZU2lZcH~cavu$Bbf>+T4L?2oAQtVcuS1kN z1bN#R$cOkeck%@XmBx}F>jAm})d9Z)zVb)7pyoHK`*JFIW_pw{ZI=|o?|T;FUm+wc zY=ZLj*jfO6`@Yl>&N82Z8TDP691%J?S)X1jm7(|v#PXK2Dwm_=WY6c?_H=^wvc){$ zw+|jWJB??4unX}_E{}#&fp{Ahs=$CJ%#Xff*SCYxsgW=@S6**ldN2>-l|_VSV3OyNaiy!5W?zYI&wNitx!pisl~jr2n5>x+ zu)15%F8`@%|Cmm}@y||=QU``Y`t_H7ZSpU%LN`Z`iPkvZC3Pk2p!pylc0kkFxbH&l z&d$#0=%@i`#4U0S^a@xm0`x9IOHoFY=>>pgi)-rTAsE}L+K`&RRPML_vy2X&lL2<4+XmNPs{I&QgejMkTSUrBWD{LE`vC^i_Y+g z6$ewxefjcbr5pZ{R1h<+@!bK&+m~;SxuEohnJlCdG04-(N#MIeEU$^7;U6tC;^N}` zn(?2aqsyw9YII8{I|g*kF89cUjmJc|F>q$o8hdJtt_fLsdokpmso#%Iq2?=mQR>pEU4snxv2rP2I97&6JYxh}ilk)Oxy?vPH zhHdv(pn?6C?qK9WrFsEPyjjCrj~7+wD}m5=G;OhJs&lI~*-FnWD{+FwFY;dJ>Yp9UdNT9Og7sG~>T-e^GIXRL6e5+zCH;Afb-5VA$ z%#JI9`C;mqg-=U#+$E5H!~x6l)+pM{0%D5M`VHuS|MY2`Q$QTwwP}KhcA%^W9F;`)O7?uiu}2?%FZBBI4n&Pg~g1emlAU_#@rl6>JBGK5hg=a zb_|H5<*&f)U_m#pxz9`#Knd$aAFHPD%cZlpdNZ}_v}A7dWIW#w zaVZ2{>)`n{H8mXOeuZGhVD=zGi$OB>Y{}Zn>d@}&2hHRMWN+Hy7T&cm{2KEeO>a;? z#;;BZDAF2(-A4x9(rnOYW^1hwD@!O)?ww-OXd*WY}mw&I^5~2-oBGtt1`M) z@FLOzmrzPkhZ%K2BcHaJew~^my;xAYWv+OZ>)2VeJo$ADUgWWH!3%v=TjQ42vZZR0 zbQLa0U=y0HPFYTWt%yJ*98l45IS3 z@S{7GPBql7NlWF+=F!noAkwNb9oX;7?c2L5z$RU^V1@|&?bgL|=ZRXdX8^)E%)Fn1 zj8Q2s4kX`i2OP4rwq^sZ_JAbp<39jI^GuJ;Wl~r}j#p+e8*{vRtxSuqXjuz1AjzlL zx#y8szWG{hfH^TBbVlX)Y9T2#i#)pBsh;EjNA(?CFyOdK_5SfqfDdZ5WyydyK7|eB z9?;99;8VTEx93chqs*ZOXy5IWz7`G??b&h!sslx7F7_=OrNhq%n<2OIpb98wkC}e0 zJQLjtM~#VYGW{cCdYU%?={}UX&or*j_9|r10uBl##nv6|EiK@^FvCTbEiV+%suBEf zDygl#y}h~FHG;X2OS!)9*yBL6_BFeWif>&Q)73^@au{FI`p?CNg)-<1Qknw#=4Sjc zu^Q+4dEAh-6wx&EN278WeYUEJFZvNOs{cjO{_3S3$<&nR4Ha*q1O0e4O&8_{c(ocSU|I5h8DBuvwFpP-W|A3hmTMVUyv76iWXLSq> zY{*ava*DCneSqB9X3?cZemOD11AZ$zTRBO5NQbz@fOgbM$}tr`;vhU0>+@>Oi?6~~ z&EvC8_i9CmD3`)kL2Dc1uwI_2y!E65d74pub+}@#E6$ab3u#LsVk_W4@G~=T`ZgZGNMa&l zXBBI0$^7uZH@fzeb@KAfjf1+zo6<7^~QF#-am78g$FAoC?4T zX#4@4_)&vAYhX|FZOA64Gv=G;kyYtS69M^1X)*>zB%O(AjV~*Z2eVA0vfX1_tOz*M z3nN!=E!LSO6cmVO@5~L&^Ch^rgc)|we3s444Y`!Hd#B%i#7LK}lbSMY!z2(psT-Hi zQHeMQj3X5eUt;0&D=498w^)SYnoyi=VT#Lh8%D)yse+Wr)Uf`0V=QnBG9lk9!FbFZfBy$?cG|{gX#A3EaJi?@c(g*Mv!www|=pcm@Y~?(9e<6t3xS}Bq4_(P2`)5U}k~u=@pZYeQN7$U2;QrbwtRcj&(`y(M@bW&p+n0P#}ok;aDBljd!d-B#01}l3$hAv)R?#+T9S-5lLDYl*G z!Vv+8G)rH9zocD}27%8=_eRVX~q+Byu70CSjIRUwNZ{-lP z7!)-GeBqY8Ur@x5egmc$?_W*1rP~HP0SaNjKP+bt9OQ79erB|AOmvPf+TisGGc&Yb z-FsTKDb;!Bc4macQV;7EDoXYZ}nk0=BoC?We%_hdDNl(#;>y#F1+f?G5`I`g^9aM zZXGNDw$g6_+e64RR``P5W3+Y4F<@h{GJoM+_0Z2y8=!d(a z9}KJ9ZNWR;#xGZ12-3Tv4#nkzY0IJL#LRpw}#R&m=>zkFY@v5S!>MvBc$NL$hTUeB3Ry78 zy$fCrtrB>ZE#!qwh4Oj8mi{_3!q~}ab-VEDgz@o&$6W$QD8$+In>hvnOnb<_3;+Qr zZ2K&LxKgZou}JTVN88PTjU`wK5#(tg5bIC6H^3%;tveW@NJ6O!{L&mLbClakYh&e+ zj|7kx|b@|oK#9dm8omso7$8xU^gCGN?G_py(Y#jv>O=B!tIZ;O?tF@ahsrooZN1?3ot425qc2 zEl%l(GLIVaQ%G zbm|S8>Q|jCcUgrOy_tYXUG8MkKId$i(mHU)&le}`avPodXDlBj@@thZ5D3bNyc~eF zxPfB~k})d*GA4Ii;>J0SD`L!Jy2A+$V)@=Xg=gYOuODL8KSL4Q5-^NAHgJ5HKy>Wf zUeof+@xWEn@nInbL~kn!V^orcG+Td_@(2~Eh2GxpgvngjwpMhdlgcp-E`DFV0NpJ> zGU#u1j3LgaLGcl=$ahoTKqP=k{xrb>?gt1pnEKP&SBmh~eayhrV!*IA2;P_fioGqC zhxU_@-ez`GESrr-39i=%Gq2nPr;~vI}-SKi+-puM}!J_w2j(XHjp$;tX9Hk~Oj3@dqLChT!!-Acl43 z#=3Abn<_4-A7bH*)XYc~1nnj6J?ScpW6VVLjX&Q5BRreGsG@zE^IGPKS~NWWxS^hb zfmbMcD~jROh2t|e?VhMDM@}5;2A2)(4?{H`S7r!1R8V^ngt}D`L!BGmLRU9#PpQ%>JcR&0A_S zH2Kq#f!9y{Lm6*QO&qwye)uN5XT-6>eOT)3GRv3cN@e7@tWe(t@;zFWkvX-TPZW3N zj!nN9M$#}&CmRUw=BxMh~ps3QtuMq^gPj*kX5q@@(&=^4odm?fe9g zN6caUvhgzt#^wO&TYH%7q9P&|MN9_uL4>@bb z*78jCog-g1CfVTIsTO8Ze{_{BC=+R6CSHi?5DBT*vHD0#SMM6}F|t?e3wj-N$l&F0 zK7-{5>IiP!xsHWpI{{)7@2Yg_^8^0SDFR_&>Bg39f}9txgw=!7%Jc43piSbaJCo_f zpW{@Q?m{bJQ%2utn3UNc{nGPMSjuVGZzbCzBb24mW4%93<4)$$3$I8^!#tIF%K2tI zb*XW8j!?@TM%eGFgoV>U5Y2_@v7xiyXf(<-+10>-f@lTnP`sS6N zci|h`1^c0bFOu=F`nt4*0jI%F)h`AzI-Yvj_~xdJqhPOBo*l_oL`)Va+(6D`E*W?} z%0?~^ovCCORU6-#bqjNiy-Rj$7h=(`Rytyrn4itMpR)va8p1vF9NQXC`(tI^eqeQ*hEWOa1XQ+MVULx#kRaA!}Y^61bPNPUJ z_A5{}9nPAK^4&44b~HU=8md@^GZ`4&_qu=~4(xhH&NwigSDqZ^YqNZe&Ciq*^s^ntK1;9;+17ICLQ;~^ zw%@Yrcgq-0hq>GlGs(4DJ`vbA$ya@BiYxUa^wnCcK%wEHCqI6`YvNdB8S$g~+F`t7 zcMLi1WFkpf)(d$RzwVECVhScJ1ClO#EcW#PckQzT&ZvTfGawP*Y(DT5?Iuq5nOk#T z=b~(Vk=>2$?nl5W_CG+`HLH8%3ul_b8DZ&pKxI1#MRr)er)87Q!=Y+n?eIePo((?Q z*f2~_B+{g;EygT_VL{!mRl3XT(j7M!ADaRFGTK_dXYPk?1CYwJ2 z-`6=Pzdap++iH|*ZfOD7E)8;8e2guuiWZY-(-X-xcru1Tn{A8Z0}+K7d|n-}pNI!w znvmE1-uwc0Y!oKP29nle%y)A67m)<(B!*fW7AkJ*j1N#^rAjGEZnlpgKg&6#%*za* zsKEogZ%}ig<;aDJ;S+b}JZR+@SRIZy);k&Qh1-fgh(r3BgLimr3Y%Z0$7rEp-eiMW z0n)eF^&TfU5+UkqBe_CZ0NJu{g0jTea*X9H8y}9ZHc^F03FDK*Y|aY%N+LHC?8<4I zVT{cs>GF!{(S>{-*ZSg`eKzrupi=@Q(txFE#FYVuTeGPVd*hstaBsX z`OnjvmCF)*KC`;G?e(&U}TSY|b|zTn9dpEl<_58nMjQOu_% zr{`HQ{@M@l&N-R1smt+Yy+R94sWibns>~Bz^d-^r5d}p9J^>UjN9}UArQnPtoStct zT{7)5A7%Qh8M54Q2TPyHXHaqjCHn*6`aO(=IyMGLL`h7(+B&0OV&Wc&$Ax+8(76&u(vS5e29|k1_Kp@z$F8mQjAi+0I0kaO}mJtV$SMVDauIO9L z<_-u;*U#FZ6L;3fEfjam15#guECzBqh@;P z3%sP0r6-$!o(vW~&J3_188;^mKrUH>N*E}mHCq*fKZnt9*f<3=H-QS)*PeM0mxGs! z2>iv$48l*H2gBFOhF%B)iL+|9ctYkUkON2 z%_it9tXXRY{X|uS>>OY6a>AZ22!w#EH9w(XWKqL{^D(MdjfyTg!ChQdKU`1(N|R!< zG1mpMavHm#pnrrWD6%>A0v8BMImF>oVJCQoW)W~c_cm;x%uHPaG(+V8S_a`z|{TOf8C%xfBqB7Akr9q zMRAbilg%uu&&59jhYRv*Gr7b)kGm^70&wDgk46dqU4Z0Zyt-91gQShL#FY~VD`Dmi zw8E?CUvEaneGp${_xsE5vKp)ANFjhV&&DYrQU4?0Fy7OFsSTicdCz|1@BPlJ+3VoF z)sAE|Ct!jM7Bl6>VUMK@rT5*M1bkEc^8?8^_}rI3PVq}!rFH{dsVW)F2xIJvQsADW zlh!yR89zv__YyGMUOgveS>Bxo3HFkmsoLyJI;2KimjN*khQTnWshL2SH2b za*B5=)bEz~zFW0s3YV^XEsuh?S?uL1j$LXDE!lbEE|s*TG`P`x5OVr~gTvC=Ov{vX z`*ZP-vm(sbqg7u50#i!^KHQ|l`4Y2{?^X&J>^S!g!B^WCUIET?cJTt_#xu}e4iZFW z{1Hu7#Vp$vJs>PSdGQ{ILYpE4K&=L|!oq|bXYbd?1gajr<2p4fcI@fYWP|?GUjLU0 zNBr+GhdFjBTGZ6o+&syv37Fb{k3A3-g9_(AYqt3mB`2Uxv1nm=UosngZPm$i@qs-g zbO6*ofW?1qvl12-20S~u99>efQPd);o~ASK*i8d8@v-mXlK&gwC(RpPbwWs!3q%L37M8zpEO6 z*Qg%51d7>(pvkLd6QmJ2L5ucjWp)A0MbHbe2S5*x0BL&Wns2T(z8vH6@9725#BtT| z1k(@ufC}eYKL-Z~+d1(|Og{{;EcQmB!g(;Q@3tK(@{mgxc74vIbY5Owy9o{ye*x%& ztgU7SY-8@@o{?Wl2m09~Y3fT7au`anRaK1$p9J~oltYQvR(;_k2|pVl_kSiTz|M+3 z{sep-0NC1NV*jH!={4~r7}!4o=zINilH;ex1#$bG!Aq6)I6Mi|Nx@+S^Kl9YUqg8n zWI<@tGY=9ymv-}Q5P=wn1c>xLPvSTVIO_-eZVVbZ-?;RO6WsCspB(p(^T=+nJQ!d) zDQAkz>$)J*F#67j#G}AYi^V{vb6u;?z4m{9%FP-635ulCY1H+8FOc0!N%=EDGbSbm z)Ls?$Je>5wgxZO_>v$q#l4Ciat&MMq(N4vg`WB7sJ8wOZXX-ZF3apC_WP#MhP?6=# zvP)ecn^s1T_UUusU~tI!`0k1NHy#TFr6W&qsbPvI$Kmr$j)#*!!q%Duiihm~8;zrY zX&C6~-PP6uX*{s#Jv8{`?(zaav84P-+-6q3NIPn{!ey$-;c#-{aE;G80P-p6L{Rhr zUDj*;7~lKP{*77R=LJ7IKf{HrBdBrEwsqiGx$x>( z;JeHU5{i$y7ED!y)j|CP)<4@kot3~)>jv@`AL&)F?rZm67|2hOakG+)%l-Ckw9LU| zm!`FdR$q-$Glg&V{Xgx!cUY6@);Ai*QFIh>1{DQ?K}Qh~P?RDq3Mv8ug3<*zNj_E#e9|_O)S(-us&9m;kZh_)54JEmMA(|vvcBE{O z#9cd^u=FL-(BrsKIKnm=3>y^r+va!6J?2wp+nbij#Z284cJs}Kk8t`HsG20-*&8y= zYp?lo?0k=l#bf*_l4$C<&+ zPrk3Feo)S3aIRi~K-%2tce4ZK`Cu=B)_xerS@PzD)=vr1|Agd$$7*QqZSzlRwo(@y z0;rtlv{c(>xjfsCMdLuggr>I-eN+gK+{L#vHrIPZsC?y%YF3qv|BjAhm3Ee(sdnr; zetYE!xOo58r$yXmdpu^Pck3Q-iy6Mj+#gcSVK<)_;TyExS^W0d_t8+ntUUQZEB#IaD2yQUiT)X5#a z!Mj!FKg8@5qcv1hg&Zr$Be*rc39_^NIN79|wa9O=(p}o;4l@*f={!=YUI<7RXc-%+ zSdpvg<Eb$6&%tL(DGEhymFOWS+0Q%hs;{+}F2Xho1sZT|8GE zvb$6ZZ@YvtHS?@zDHfdy+Fj~SC%7CcAS^8@U@`Q^^o|ZeZ+~4fAD*!)7Pq%ZJy6Bx zyY=Ec_>3(@06)kl=0NieNa43#T=mOA?VvAf(>TV8WD07Ps6b$mJ}>0^u9-D{VK@=SV_9rJL#Y@uyK!eAShcR$xQH5eONPi>P~JRwje;2Txssum-0|KE?}esT(U& z`D$#>HevI$Vmmimi_IDMe=&7ZP;&&^DqwAeT^YR|;Ug0Aw6k4)8oX8nO{n0H-I9t0 z0}DX@5tKi4J^>y6feX#4>JeLa9elpwpShUGZ6A7n&q-2wwd<;+V^@(??b3{bMlgY; z6jPvs-x0XgoBHH^)q@@X2TTE;c%S&eM0V*X0vlx{r&~6PZADWfFw9$@`fT(~msrFm zA`LUy8N{tQyO!bDiC!DWnRUJE=B}*Ev#fsTO&Nn-K8lq99w5cpQl@#;s z#%LOc5;FVe=J*9mM#Gksa5$m%de&kh8BIkYOG&jD)&~@RkNe&$2z5X@zWx-FrWqsS za_P{z<0qC=oD5gl@b?k6fqw>bx9j-HN-tS)Ie%guuxJdOY-R0J$uy%6#t1(iaA8#= z6TZB9GP4Ql)}29w0CFLG>6!6L{7OwC`32q zBE+)C?;Ty&R+sg^xZJE2bY@I29Jm6$8n!K!s|&8JaG)-Q2RMXBUsyJ?SS&&wyroX^ zWEQ<%o*|=aZPP2R^Qd&)VU080-t`h>^X!Crf98D@C1z+~U;x1~2pNDU5G(^g184;x zH~$lY@@4-XsRXYn1&g)*^gf%(CumxKWUug&-Pw;b?Oh$6+6 z!Mwiq7vX(NIaZgB-f0YAHP~Pn_4h%A8{_@lkCEMy>6;rBgCG9E4ky;}8c7KW0V(TC z(2TJMFfVC4y+JE4_6Y-bs|Zzx@IW@sjobfd`hecuX;PWF&t%}-AVnw$7V;rLR*%_{ zZJqV^FbCue@|(v(Ayx|)s7**@N5u%#6)>4v@w<{_T^rAgAME4=gj}Es&haq8WXEYK zfR>GmT`Z4A#6od5G|xT9;Zsws=LX-En8jcGd%y1W>j@iPQl;fyfDRy#nN0>qxYqqX z6gTu;&~0|zX^yZ>1tYNTqq56CR9P`+LjQfR^TZHt1{zTwyp;(8LePi1Y5a6fc&fC% z3($C_ecQHj(gwT`b`99A{8P6+8x`8?575>t1oq3{&GcKJxuq@54)mckc!#in@5D=f z%yj3HQ77`0L&ub1|MuICNznAnh6U%Lq7&c&3=R)lXYby_6;`y)fH|YtJn~+{Me(!fN)fg>_(zEJa`IIQ_}~A8 z^9T8V}=1 z@OVCd1AR!4=&{h%g&HJqv&%eYHo#7V+&z%i)BLPlH17%pt7OK`8!-K+|BAs=TCv2_ z#a4B?Jp4w431QX}aEkg)cJz^kcd$mSM%6692mFuZ0ia~{l7s{t9{hwDivWPwb@xEW z!to8;9Qcz6Ko=TN3oeT)6=vj9A!FvS%IcH)DvNtmLYs0;kvxNC<;QdF)5rQxNrssZ zAl*rTV65f7j;OK2pEf_!{l;8A5=2T|P0uhvOGZ#T+V5KmI89&!s6luzvFjujv993j}4_o0uI z;;0zEUM!k6Y00US{*&B?wtrKw!*)ZTpLhu-;G33C*z;hQu*x=mwE&-~k-dbKmT|&- z#H5Z>6MfZMJp-5OjEB@u@x00@ZWmfNh&9Bs>zF1eH-`ZlIP8!1pF9+r3}_X`bHMwX z^d;Qg9t7a~m!)iSg%W+B*W7PH2OU>kQ(oWs!PGygk8u2b& zA9ySj^M>OV=aL#CCV!C=0{FG$>`lm-QmIt$n6Li`(wKE@6L@yy*%i5`t*?oDWdo)| z`M4@_u-!Px(+KlRG;ORpN zh+WwKbL6SO?xg}7jz?%;4SC2d?Td5jFLLY5QKH*ZP+KUXPQHnq`C2dIeqw5a#+G>U z955ef5CTL52t%g3a&`A;CCKFhQvCg?W`7X`T-tk$X|cA5TSl&Yv}(%fDPnm!yY^R5 zm(9@0^q+QhLyhyFTXqbyVM`q)>a_vd@bEBHB%jx{UUxhZa=RXI3Qu(^&%JU$(Uyj_ z+_(7ebh-YIbpI3}T>mcFk4Sh54sqLYdP@+=8GW23oOJf)Kj8fTRaPF6(9O1DtkrS+ zYL9ht@r7V$|0rce7;L>Mal<%VV`tXoJ*L0;SZ7B- zW!)SvK)w%74)H*h%%8&02i=>GjNi5Q*!k+`xvkUS5<`J|?By}2y|nnQjo~C2t*XJh z#6=gv*Qx7|u>(Gx;}+%-WugL$A+bAHzFur#x?BU^#5!&ZrA0&nk|neVn<5JuFf&>K z;GXHl>1@3J7f&JV$_2-$??Q7FYr(kg=RND(2%lpC_{h+~;DMi{oVo$L1V&^6()QBL zjlT0hOUVtLdlzT^1K0nqOyRAW1H+1ThM1^mPodlM9c;02xdXv2J3QYern0RePFk!> z^~HSmzo1XDb}iP>Q!sf~KJ<_(8|#4RiH+@5{(874cyBKh7})D;c{vOT{A`N{BM8>; zgKOWMP)L5Gd{%imlw^X3%kD?raGwyonJ%&CT0@unyC~yC^qJ`-5AtT;mI#x*)9Nni zFcGXZ;f&k5ctIOagDDZJo}pa=x-l;_gDUxE1qS8zxV(62+|5Jw>h&iVdoso>&(XAHwctTI_BS)2;1LR%-mu;|?f?%LG&qi~A24eI-CH4>r$F?Z{l6-d}`&5kC z+$d}LC8}|Ss&Q*mD>$h++K%i`d&AZS*lVCSeodlkW{)u_7p5xevaDMm==K@gQ-T45Co+H$fJ$T zCF~a5MS<8CfdM;;9kEC+? zf0A}Y$b~>U#IbJ8b@t`$WX6+tW!5T}Igi<0gR&~jolU~3u({UrMj%1Z6%L?@$F$dI zl;K6)yY2dh)Y*Q84fBrB{`EoFFHG9X2nXSWxgRsbR%xvv%yjRx?w1pQDK#lA_q~jP zdyy=kQ-Q2lIQ`)_M_MAnD9>8`A1V)Ia-ul}s?~k^!q+)%FU$Ar0Ay0}k_Z~~v(Qd=l<}P@ z>=XbjZNi4#6T*hSA%_-m2+S#1aS$#DtrnDX01O4Y1vUg)f?#=2neOb{3_NuWJlb<9 z7u{(+Ztv4e+RyiJqY$4pQnVd*IQbRsfhy!a_p_07PXUk95y*O*)w^ z5#7>Hm(XfZ;!LW7+U&}L8Ikq1Sh>jakO-RAiqUS%ra_br0qOimt(tBdq!GE@ff|~9 z7q4Cno!@q$|EM43rEm>#zAijq)Dy#p99h~?Yav@Q4XNYCsUEx&Y2OjfwGMB3{?=ub zVb{eoo5ime&pkT+uX$C3_6u00e@;OGhQuMG^t6N&^4YA`CyO+jO(4^_2Y$CL1+Yng zSOxdRp8T+0VbY;1!oD?xusOU$Gm>FNdF~C*;Bru=;iw@F4vrCn?Z3#LX)e%;N%Ge)ExK)@*OHoAoH@bi*iG!b$9*tZQ=-Hg z>ZNK+VXna98l_3{JaYYXUqQT|ueEZ@)C^g(;RV9{Bh*3CYqsfd?9SWGSpU%zD^uY{ zqZo4^J7eSSlj0(xqO?vlqqC}D9pI=uSGB#-+OyA94Nw|hb2PO0SpYB90}{Jt4iWf! z!r*5az&C-`s_fo);&q6!!27g)X}TU3o|SudV77)2_1Rhq63j|I-TH=R(tu%G@#e2z zdT?c%up9*(rg`X!ESs<4)6~!~XU%roVi@tnwWIM3MGl=3s{`x5x3NLjdGXLgKHg&Ldh~`sCAj=~F%?YAeGV?=#XLZPVy@o_FwA}wq8h((K?@6{aH{2pn+jQbY z<;S4kfBz)?$CYy@&sFU(cqpc%lBA4&~Br|9<}^S82DhmD_gl zIl3lxuZ$-|Cx7`%Br{X)z1Du-dzLP@mcofIoTcP$G}%@XNz_hD{J3X8@KBR1(JlG= zOwRA~JAQnzAf1H~iQQ2*eL%@c)Q1@L68m>T{mdleQzkef-Y{ zni!KZ*AYPzdybvFiq`LgUezXgweW>2XA;Fs%dF~KmxsB~fiV3nE1d5n$dUODH}@Ojcmr`bxkgMi(uwO2($gQ_(GipA(&!4Adwi3mPZ zO>gI!DsljmBF=-hn+@tm-%DO)YO;o^ObrdI&SN)mJQ9?>bA=e4uCJriQ9N0q*CV6o zLLHZwW)w;$$&`B3s?m0O&sYXiCu@3|a(ui-(;w?;5CgDt4eM>nhzU#my-*o9r@({8 z>K*mrQea|{t{tYroKtIC3S_axBfS=7(>!9hsE%4}*-Cr_em*yaHrp}m;7n^V6tzk2 z9PDgjtPt@dnJ?X=gnKdGbG0!6t7U}#HeC||JS{IioHhK4!z1AKm(St;-edLHiuuQ3 z$_r&tX^{^Pko|j~_xJ2uU*u`0*OMyKRNQq+ejz{17DQ4L!jS$`6r1G8maL&d;fAsm z?pm(nX_0wDF^0qfdtArBo#l3-X&(G`DkFiB)bH{Z7dr!JFF>^rii=C zYzX(5?&xVC;DiQVi;+5kb`cD_oM6;3Lrj-@rh4C994Gq#C}j>0&N zor<&f(aMvvV$nr0OE$8MyWMtz319jP4H+vv3UHpZVb~_#R%;r_bWUR z56dUT)eHiTkoHOVCHFeWN~|=5!fUeg8cE2Zy>m!Upbg-f!^OS8L=F~LNqlkQtIxKaYlHSf63~`nYc3Jy|M+!06jJ2kv#W zf^d=LAN_NKG3V81GD33Mg@uHI2k&q@P-x(WxcVEmk*|DqmWn@e^H-8qdq5ehHUHe-~7nbS?e%+Eu z_TEFQ=~PphY>U+|X0k+=i85vPfmp0yTlU%J+&fHZLeDh~k^8q2SC^jlhx0B2;td2? z#jWdb95m>en)X71y0l_r=D=ErzGlMh?xMpyi!)^;%2Qg$Ofj}J&BoL-dZ5VL&2(JN zl*i2H9l8(ya?yVOnbk~ntB13Oy)6aytHT%SJ5ge3DXT+#5^JA3(4}7!M5vNw6VYXO zR*JBKhDc#>gZFY|fAP7p<3?`>UuWe^O-%xzY+`DpU^pKLGjetwC=!OEO*D&NUuw~CJRba6B7-r~VduA&${jaa!6&i$nXY#{y zyV{+k#^nuDf=@FUK7ZiMt;;_Ejui5@SL~8E&B)6FRaI-~UUb0n&o4f#hH9K7Y;mG5}<{8x}bGPYvwQKg(pDYH- zQ{BfSWR4rnx!7FuTuGfbds8s{GA$9O3zP-v>1D)?>EQHdQz6_oU2i-;XR@c|%CY#6 zKBbH!+&u;ZUaM<0dKx0s+9Vlh)Qt&f2+6>^vfvvM+Gkat&`WDM(BO#%RTEVvv9(dr z&K{G9&3FG0DjlWEc%G|T#@c}9-*UZ5mebvOa2elPyDX6w*%6GVg!3@w?olPnzkVro zZsw9)oAwl4)>=G(9QQxPT9KMAlk{#%2=ykS>$*j5Q-81&{-bMQ%FEfl280Pl zh`3IFz$7w&(fEjMxOOQp7E?&2U6xi>R^zv!#>IEhbNwv@R+y8M2F~g=WS+sSPE=!P zINx{;dJR2?))$;381S+9{xY_Apsjc~x4p>&uX*xzO*utdkoxRYR1e0XA74=ssaJ%n zSj-NS`WmT|ST^0tLo6+=P^5o+WFbfI_%&ES+2aynX^bez>bD}9iq>9gpGGng2x#A! zqxfd_9#DKdMjvIo8`mV{5ah9kv%fsfZRY~GXcZ#Wo9kJYJ^%;-FW3$$| zQd|;9owr*&3#Gv z@e1Us34J1gDB!q$9mLWm{e8k3BK~Ul)x#D__8n%pevfe_%_O6IODoEfGUxqW`;}+^ zIv|W6O+7He(J?#UQt?pRmxZ)gN`HVOdW@SbOIj2LjHHY6`}1wxbwGOl{d@Oc<{zbo#9TVqhHo5%R6T8!_=nwv#{i54Ss5~biq z>uv12@5JdpZa7A8<5FH1XaxP4ed%t#^E?%n_R#2pwdJ)1pNzx8bvSSKO#6TYH>!Scm`Q$LLSug$#?L#a`tfQQrop0Zsn5=+* zIU8w}h(P%00Y9t`ifuev0J9HFIt^_JNds(QO#PvTVHIGVzUi?4W;Oidvh3AI{8;t% zW%wb58~*LTiUW0alJjgERJ=ne4Nyyh0_+7!@1%isbndM5Z^D@pP)7sh#m;VSZmO!P zI^^|1a4$5@bcEPnyLz>?b-nZok!e2sbNlVez)eaO>puvb`aX_b_uCA>-T%E`n7Cld zfY=AGe*WCG|N1S=p=bK-Zs@mH$PT(+x}>sS{ zt0;aZr0K%t)mp9{%d0bmGUqkEE65)oKwtg;c@adVySF?!4Yjh@Jv=<}veq@%4YfPK zT@3Z&Uxm%j%jHr^Cz8XSnQeHx3eSPSwHPWzk@JVDLJ}_lo&KFWcYtAhtOZTMGnMSEM}v57u0uqKr+L|}+O~jUN*+6Q$zw0VvCOUUj4%?3 zY{q|mrYV768UEU_xpJup>O%o0#w9BN{im`&?GN@X&V3TY_Q4#9z5P)0dK_p{La2yq zORdC*{6X9bFQIZDc!7n>enAwORrtj7D5qy+#J6vnDu@1~YqK);Z6CjKT^W~#c_4Ri zPz=asMlv{w=)DiL80;hP()Mp*)PL>6hv9jzJO>0KAP7#%o(4q<~ z1;i=)`&_`WOWot7HE${*&ffsq2n)aY3l}_rf&)1Cwctx>HaiphIRB{#yiGXHrUreB z>HljV^dbOPxFUfMc-N+mEoYF8-3TV-HydcfK9i4lZJ}De-kdHlfF<6W;r;Mhp+pYd zp@1r1ZEQ29P+-uX+4GiTW!(}^rAzXQf2D-?p>cm_GJi${q;{UgVk(x9!_V>bBIg?m99<%BFdoZuNbT$UfXtHgB^br`Rx?Ufi+Ps+ z*Ra}=x|~gjZ4W3G{u7NH=5e^3(2>|PwH9k-J7`7fy)Eyg-D+8kMm3_jQF+?^SK?fX z;|msC(@FX>O{R$RSnFnml-bhdZXE-so`rd%2qhib=%^}}Q&e7B%29M+sa3Tax?-_3 zW})7vJ&5}#`o%8w13(Z3ZznL(thqen;&pfO*CTw~{gDr| zg--j)}^&a;QCoZ zuf4Vjou2UOTP5)N-R%r5LE@S5bylo}n2cN4?LzqJegbmtVv|_-=ht_O>I7fx5=mH{ zQ<4sM%NWK>nH_ClIJhJxc>bS5;fA|c}bnp0V z%kp#o-&li^(07o<`|3a-=9X%;bL}#6c0##=ORk65rqP=dT)~}6oTw5f_CrIijPKX2S+2*t|RQvR#(utdL2mBGS0`l8?tkUI+2T2cG_ zxVeOap2S78=EUWjx_=V!UG#YOC#uLW1kv_(r7t!=l)FLOtn6A}Y3$mjsi0)z@Bzl! zu-7(`Gq&a)7i%M@)Z*9fm0kknSxD@`nbTjK4%)49|325OF8*Qi?V+l?_sQPCi<7;H ztE}Ky?;d+b@`J3`eA~`C>8`AH(J>YWRkjI@R5H~J73ua19GN>pk=~251|r68-7r)Oux}m+|2T!#}DsT`IO@?NpEl78#ZjGr9nvx~41J$27A%}k zdHeRZ@3rlfI+XTel5>2qqPa=AWkcv_R@iTZvuURr&kTRwv7kL`zM63)0$KSu$E(rG z6IpgbKt&_CRZ`)UEvY#O#4rWK@QUw;9qCMJxF_ClHZ~3Qw9uyW8HIhRIFvkMy^D7?9Q>XK_`*0sTQQh!Z3Pr;@f@P%G z3-DMy7}xF>f_t={uicaXq#%rV5iM$dV!i|BB#fdoRJKuj1<<2{O`l#dFQ&@dWL2sj zzgBiZ98svM{%+b(0KfF7TQG5Lg&r1$ql&^l>bW(NP)iO-_iKOuLc4e5rDX*2ZS~tb zKOyX2u7q-_^?Q-7VzBT6{?^VgSGqZYHzrn;eEfL(N{6U~=&qM>UvFBuu5y_k;FNkT zu=Q---FjaaKl8s(q7o*!Ta#6i$kn;*?ULop)hTxhk&lvbpmnPSdL-7YehkH4y)3WU zi872aEKeGvvVN;O(=zL*3R^fLqhS4d_xMwX8weFbPcOgE^BrY0HFy(sj;f6IK5Fp& z;@tf()Mk)V)Y$=%>3);iBE>69^bgncc*(9yQ9RR_72y{*T9Cp~4PRkN`LEXX z3D5H4Zd~0)ElcqKV4R5VGd~JasGXP1jcDVIUmzBbOeqkr=ZSzW5BTz+)4ZAR`UVh* zwDc6&9jJP?Tjg9pfZc@NXx`k)v$eDkgO1rU*_`gUOjSRC{><;{2W=#IX;8X-uKM7kdSGT9YI8wHsuRcG&P zRlikJoP~F-sU{edf+gX#Wp^I_f_ei3xgsZ`@{!{roG%yioABF$-Cx=F#b5$VJmOnl z`j_z@KM?4&mslJ?gCn-W zwgc(qI%KdFyJp=&!Tb^Dzl6`Ucno_a-VtE=!gmji4`W%Z5U|Is2Rey+`ugG~5 z+kN6xGlrtQ)B~C3fJ?dL6oOTMOP6xCU$|m$yG;=9Pk}B_)_Os>_ytZL< z{8sDlsmePK6?V^iOIAopHh(^o>RHd51&`WMq#5(~+ZQK-uFMVeV!qDg+V1Ujiwux* z^H5JNsdp}Gm!!AB!)~@$n2;RdaI*QlNJw;_34$j4992QQQ>6q^{ub!z-$76R-fd7p z!4dGeQbD_Ur`?$P)jn%~RalxXDWrK0tM@z@A^VQSYZ3Xbl-}s87~!t3ImLBgCEq~$ ziUb0w|0%$qOVT&Vy*rHS@6}|luoMk^0IgYF?6;X37;Tp=$MzDp$HL21gO^*UFF#(V zOU?oE>^5gxE;4bfmeJcc@#S5Hr#2Y*;qJp&{Cu5Vy$q{Lzx*yOLiYM8sa%pHqcMYX zb|lIRgkuVCo78?-3A!C?npmzg`pa5zWyz^6^DXTb<;=N4Ll!yVeQHvR_h4mGlz|~z z098P|8$&*f%v*QD1le3^Z2yVHMy^nxCQ7-WT{fLfF7A3m#$x4jTY6`PS0k8yAI!+u z%ZpRej_o2D?yUun#CQVEpyqq>gkR_RvX+X(!X0xT6cQ1CsQFEr;>^71rmLgj_!_A= z@&NdM4_~E`4A;gUSjuHFXBRwd)^lx@8b>!TgPD47bE8w!FF@ zgJ=`Y-RGZETs(h5K=L73QDSyKy-%Dq{4$uYvQgKydpQ zW~=KG*3e4y3i?zQN;`J7{n?*;9b`IC3H{~z+TjCg2i@+H8;$kL?e+#*hf0&qOvL$y z)BQe?P?uv*j#IEaJa3Fmjh;Pw+Wl5NtY5*sH>-1qL0h~XYu-Ky3MTOhJew+e`>>Dm zpLVQ|fB)YOxBmBnwEv6$EA1$tNHF=+>YXMIxi}EF+vCje@Q@l#!$TTkE!*!QAR=?G zf%09Ie2VaU)d>%(nj5q2RQD)^a)~yk5L*TYtwh$5eSM%u{1A|;0NwA3OcS zClXi-_bSS!d*qak>g5J-mrQ1p76!Y1=fymgoogAt2!Tt`+ZDm^%f*!E+^32>hMslSDi)@e=hwgJW11=f zCgE}2fW6FxSAeCWJ{~wNcaR2X4A;&0sLOu!54*V%mFA+m<5iBux;53QnJs2p51V#q zv+67F;n4fAi@F*n7k$`=iN>4Zd?O7L+ATYgcr{z=Ye8FqZIDZ>#yoC$211|RQaYdc z+VaZt7hXLbF+fqdB)r`}w`$faI5jpZr3bjDYy#&P$f*(F%e1wH+6-KHum#Hqm~#^3 z39j+QECmx74lTZd4J)@lIq39zHXQJkekJ#E?tVO$)wzHso|qsU73a#m#j%q)vbm{T z#=Gb5@S_A8))y=9)Kn9%rj56JUrdcBOr-{Q?qeU?tE!!$O{GjrjiFm99F78s(a{D2 zeQzxp)4tALcf8?UeKzo2lwbR(96a6W-uXH53TvYL9lvOe68+=zQPWjlP4fekYagCB zj*kgUnR6lnhRLxi4OfpY!L~Dd0l27=yBtfByR#_EbG@f% zD$6-RZfr8tG1+(aiFvOcOl_W)Q+xz(y-g*mI;14uw$|s-e%6&V>trCFFtl*AKh=`5rhQb1FH+2zNITgS zX&B+*IBT5kkSqmf1TY0|+{v@@9f!U@sRgJcLOm)r?|L)l#2KXq1WZL+e{t|%oA)(z zE3OMHZxvm5@*q6i&t@e0;$FPmt+lB2cBet!n@>Bkc-1+srS+(J@Z-tuwrZy_!Oc&E&O7$~9F%XC%Y)!^k;y`&h zrSAF3Q{36oO@Djmo@l;&=FoD&cA<|x*M@d(r zwNeE!3Yq$M*y5AAZf+b=)W`dyd(+BK*-m%G3LU3+a&w1q3tA_g{<=7A9R+H=Zryfu zF_#(?Py{1m-`TF*R!Z&Tq-)P6ZgGB8N^@;tzh>+2#tV?>3%>2I^`-q+6DSLhkFox2 zVtIG!>y+z8YZV-fr04$~$CLpf>t_oY0(~oJoy$vF3Xsdlk%v!p2y0XC$7_`D)*_i^Mw>03_|x8Q z(!$d%$KBkx+%2{uDz$4_KZ6DUMNxsz;Jj_#ES5n3>aNNmfc~5-4D(5Fhmo&CTn^WkWJl&jlZ5-~3O3Dl>R;MsCk8upj^$ETHMA9F}L z#!W nweEjKvh^Yy|F8Ve#@dwoPHtwuE_V_88PqRmpU+Xb@#p^lhGVPr diff --git a/report-example.md b/report-example.md index 84981b7..59c93ab 100644 --- a/report-example.md +++ b/report-example.md @@ -1,6 +1,6 @@ --- title: "ML Results" -date: "2023-01-31" +date: "2023-02-02" output: html_document: keep_md: true @@ -16,7 +16,7 @@ output: Machine learning algorithm(s) used: glmnet and rf. Models were trained with 10 different random -partitions of the otu-large dataset into training and +partitions of the otu_large dataset into training and testing sets using 5-fold cross validation. See [config/config.yaml](config/config.yaml) for the full configuration. @@ -33,7 +33,7 @@ for the full configuration. ## Hyperparameter Performance - + ## Feature Importance From 3f91d0d1540a9efe32325af1777ea1e6599ffccd Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Thu, 2 Feb 2023 11:58:04 -0500 Subject: [PATCH 52/53] Try rf & 100 seeds on GHA --- config/{glmnet.yaml => config-gha.yml} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename config/{glmnet.yaml => config-gha.yml} (93%) diff --git a/config/glmnet.yaml b/config/config-gha.yml similarity index 93% rename from config/glmnet.yaml rename to config/config-gha.yml index a32c43d..129e4db 100644 --- a/config/glmnet.yaml +++ b/config/config-gha.yml @@ -2,9 +2,10 @@ dataset: otu_large outcome_colname: dx method: - glmnet + - rf kfold: 5 ncores: 4 -nseeds: 50 +nseeds: 100 find_feature_importance: false exclude_param_keys: - exclude_param_keys From 8a3c44623a6dd7681bb49eabd771406e2b98b82a Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 6 Feb 2023 12:38:27 -0500 Subject: [PATCH 53/53] Print wildcard pattern onstart --- workflow/Snakefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/workflow/Snakefile b/workflow/Snakefile index cfa1c75..eec64ae 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -38,6 +38,10 @@ if find_feature_importance: results_types.append("feature_importance") +onstart: + print("wildcard pattern: ", paramspace.wildcard_pattern) + + rule targets: input: f"report_{dataset}.html",