AudioCatV2_ML_Framework/dataset_loaders.py at main · JsGraaf/AudioCatV2_ML_Framework · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
import logging
import os
import sys

import pandas as pd

from load_birdclef import load_and_clean_birdclef, load_and_clean_custom
from tf_datasets import build_final_dataset


def get_birdclef_datasets(config):
    # Load the dataset
    logging.info(f"Loading birdclef data from {config['data']['birdclef_path']}")

    birdclef_df = load_and_clean_birdclef(
        config["data"]["birdclef_path"],
        config["data"]["min_per_class"],
    )

    # Check that all the paths exist
    birdclef_df.apply(
        lambda x: print(f"Failed {x}") if not os.path.isfile(x["path"]) else {}, axis=1
    )

    # Check if the target is in the df
    if config["exp"]["target"] not in birdclef_df["primary_label"].unique():
        logging.error("Target Category not in df!")
        sys.exit(1)

    # Create a dataset containing all positives and pos_neg_ratio times negatives
    logging.info(f"Making the final dataset")
    datasets = build_final_dataset(birdclef_df, config)

    return datasets


def get_birdset_dataset(config):
    # Load the dataset
    logging.info(f"Loading birdclef data from {config['data']['birdset_path']}")

    birdset_df = pd.read_csv(config["data"]["birdset_path"])

    # Check that all the paths exist
    birdset_df.apply(
        lambda x: print(f"Failed {x}") if not os.path.isfile(x["path"]) else {}, axis=1
    )

    # Check if the target is in the df
    if config["exp"]["target"] not in birdset_df["primary_label"].unique():
        logging.error("Target Category not in df!")
        sys.exit(1)

    # Create a dataset containing all positives and pos_neg_ratio times negatives
    logging.info(f"Making the final dataset")
    datasets = build_final_dataset(birdset_df, config)

    return datasets


def get_custom_dataset(config, percentage: float = 1.0):
    # Load the dataset
    logging.info(f"Loading custom data from {config['data']['custom_data_path']}")

    custom_df = load_and_clean_custom(
        config["data"]["custom_data_path"],
        config["data"]["min_per_class"],
    )

    # Check that all the paths exist
    custom_df.apply(
        lambda x: print(f"Failed {x}") if not os.path.isfile(x["path"]) else {}, axis=1
    )

    # Check if the target is in the df
    if config["exp"]["target"] not in custom_df["primary_label"].unique():
        logging.error("Target Category not in df!")
        sys.exit(1)

    # Create a dataset containing all positives and pos_neg_ratio times negatives
    logging.info(f"Making the final dataset")
    datasets = build_final_dataset(custom_df, config, percentage=percentage)

    return datasets