Merge pull request #16 from ONSBigData/i15/add-index-generator

mitches-got-glitches · web-flow · commit 55f193b48a92 · 2020-11-19T14:15:01.000Z
I15/add index generator
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,10 @@
-test/test_data/*
+# Ignore txt files in test data
+test/test_data/*/*.txt
+
+# Ignore big data files.
+test/test_data/*/huge*
+test/test_data/*/big*
+
 .cache
 
 *.py[cod]
@@ -47,3 +53,4 @@ docs/_build
 
 # Spyder
 *.spyproject/
+.vscode/
diff --git a/test/test_data/aggregate/aggregate_hierarchy_expout_level0.csv b/test/test_data/aggregate/aggregate_hierarchy_expout_level0.csv
@@ -0,0 +1,7 @@
+,top
+42736,100
+42767,101.5019018
+42795,101.8915153
+43101,100
+43132,101.1929564
+43160,102.0133392
diff --git a/test/test_data/aggregate/aggregate_hierarchy_expout_level1.csv b/test/test_data/aggregate/aggregate_hierarchy_expout_level1.csv
@@ -0,0 +1,8 @@
+,top,top,top
+,cheese,beer,wine
+42736,100,100,100
+42767,101.1823223,102.8954971,100.8805763
+42795,102.580723,101.7657382,101.4316824
+43101,100,100,100
+43132,100.0827551,101.1756397,102.0357088
+43160,100.1436985,103.0486732,102.8116249
diff --git a/test/test_data/aggregate/aggregate_indices.csv b/test/test_data/aggregate/aggregate_indices.csv
@@ -0,0 +1,37 @@
+period,index1,index2,index3
+01/01/2012,100.000000000,100.000000000,100.000000000
+01/02/2012,101.239553643,96.605253238,97.776838217
+01/03/2012,102.030305330,101.450821724,96.591018620
+01/04/2012,104.432402661,98.000263617,94.491213369
+01/05/2012,105.122830333,95.946873831,93.731891785
+01/06/2012,103.976692567,97.459145681,90.131064035
+01/07/2012,106.567686782,94.788761174,94.534875220
+01/08/2012,106.652151036,98.478217946,92.561656277
+01/09/2012,108.972907308,99.986521241,89.647230903
+01/10/2012,106.201243857,99.237117891,92.278196038
+01/11/2012,104.119138987,100.993436318,95.758970985
+01/12/2012,107.766009780,99.604240113,95.697091336
+01/01/2013,98.743506983,100.357120656,100.240738302
+01/02/2013,100.463054311,99.982135132,99.499007278
+01/03/2013,101.943121499,102.034291064,96.043392231
+01/04/2013,99.358987741,106.513055039,97.332012817
+01/05/2013,97.128074038,106.132168479,96.799806436
+01/06/2013,94.429441620,106.615734964,93.720866546
+01/07/2013,94.872365481,103.069773446,94.490515359
+01/08/2013,98.239415397,105.458081805,93.572711493
+01/09/2013,100.367748271,106.144579258,90.314524375
+01/10/2013,100.660205114,101.844838294,88.351368484
+01/11/2013,101.339483848,100.592230114,93.028749289
+01/12/2013,101.748769823,102.709038791,93.382779332
+01/01/2014,101.734394910,99.579700011,104.755837919
+01/02/2014,100.247760523,100.767329610,100.197855834
+01/03/2014,102.820802456,99.763171909,100.252537549
+01/04/2014,104.469889684,96.207920184,98.719797067
+01/05/2014,105.268899775,99.357641836,99.997866710
+01/06/2014,107.416492043,100.844974811,96.463821506
+01/07/2014,110.146087435,102.010750298,94.332755083
+01/08/2014,109.170684841,101.562418115,91.154103517
+01/09/2014,109.872892919,101.471759564,90.502291475
+01/10/2014,108.508436998,98.801947543,93.974232244
+01/11/2014,109.912481180,97.730489099,90.506382342
+01/12/2014,111.197567036,99.734704555,90.470418612
diff --git a/test/test_data/aggregate/aggregate_indices_hierarchy.csv b/test/test_data/aggregate/aggregate_indices_hierarchy.csv
@@ -0,0 +1,9 @@
+,top,top,top,top,top,top,top
+,cheese,cheese,cheese,beer,beer,wine,wine
+,A,B,C,D,E,F,G
+2017-01-01,100.0,100.0,100.0,100.0,100.0,100.0,100.0
+2017-02-01,100.39114570243672,100.53722757000607,102.16271393015663,100.46918102520576,103.704269157839,101.2097615215646,100.39945936804897
+2017-03-01,101.8817734660258,103.48012177385716,103.17968630977327,101.9950071204797,101.68931522100627,101.70198827062052,101.03661992364385
+2018-01-01,100.0,100.0,100.0,100.0,100.0,100.0,100.0
+2018-02-01,100.94149900558011,100.93285239492748,99.15994943559706,104.90335365546058,100.01072913633199,101.37445964659365,102.96145768950983
+2018-03-01,98.70067578374201,102.90956243773932,100.72644142121007,105.66479675637001,102.23113457039486,101.73431437416218,104.31985951638451
diff --git a/test/test_data/aggregate/aggregate_indices_missing.csv b/test/test_data/aggregate/aggregate_indices_missing.csv
@@ -0,0 +1,8 @@
+period,quote1,quote2,quote3
+01/01/2013,100,100,100
+01/02/2013,97.02107546,96.33756279,96.85624492
+01/03/2013,100.2030568,99.38630881,
+01/04/2013,103.9939693,103.707301,102.8423593
+01/05/2013,107.5557316,,106.4030102
+01/06/2013,102.1595008,102.1210296,103.6548477
+01/07/2013,106.8323137,105.7164724,106.2376498
diff --git a/test/test_data/aggregate/aggregate_outcome.csv b/test/test_data/aggregate/aggregate_outcome.csv
@@ -0,0 +1,37 @@
+period,index
+01/01/2012,100
+01/02/2012,99.22169156
+01/03/2012,100.2919024
+01/04/2012,100.1073972
+01/05/2012,99.78134264
+01/06/2012,98.47443727
+01/07/2012,100.4796172
+01/08/2012,100.7233716
+01/09/2012,101.3165451
+01/10/2012,100.5806089
+01/11/2012,100.9697697
+01/12/2012,102.4399192
+01/01/2013,99.4561789
+01/02/2013,100.0865296
+01/03/2013,100.0866599
+01/04/2013,99.7722843
+01/05/2013,98.35278839
+01/06/2013,96.00322344
+01/07/2013,95.96105198
+01/08/2013,97.82558448
+01/09/2013,98.03388747
+01/10/2013,96.93374613
+01/11/2013,98.59512718
+01/12/2013,99.23888357
+01/01/2014,102.2042938
+01/02/2014,100.3339127
+01/03/2014,101.4726729
+01/04/2014,101.1767484
+01/05/2014,102.5726957
+01/06/2014,102.9223313
+01/07/2014,103.9199248
+01/08/2014,102.3992605
+01/09/2014,102.5496702
+01/10/2014,102.3533384
+01/11/2014,101.8451732
+01/12/2014,102.8815443
diff --git a/test/test_data/aggregate/aggregate_outcome_missing.csv b/test/test_data/aggregate/aggregate_outcome_missing.csv
@@ -0,0 +1,8 @@
+period,index
+01/01/2013,100
+01/02/2013,96.7763473
+01/03/2013,99.89606948
+01/04/2013,103.7481328
+01/05/2013,107.3294394
+01/06/2013,102.3447665
+01/07/2013,106.3897235
diff --git a/test/test_data/aggregate/aggregate_outcome_missing2.csv b/test/test_data/aggregate/aggregate_outcome_missing2.csv
@@ -0,0 +1,37 @@
+period,index
+01/01/2012,100
+01/02/2012,99.22169156
+01/03/2012,100.2919024
+01/04/2012,100.1073972
+01/05/2012,99.78134264
+01/06/2012,98.75024119
+01/07/2012,100.4796172
+01/08/2012,100.7233716
+01/09/2012,101.3165451
+01/10/2012,100.5806089
+01/11/2012,100.9697697
+01/12/2012,105.2864531
+01/01/2013,99.4561789
+01/02/2013,100.0865296
+01/03/2013,100.0866599
+01/04/2013,99.7722843
+01/05/2013,98.35278839
+01/06/2013,96.00322344
+01/07/2013,95.96105198
+01/08/2013,97.82558448
+01/09/2013,98.03388747
+01/10/2013,96.08353503
+01/11/2013,98.59512718
+01/12/2013,99.23888357
+01/01/2014,102.2042938
+01/02/2014,100.3339127
+01/03/2014,101.4726729
+01/04/2014,101.1767484
+01/05/2014,102.5726957
+01/06/2014,102.9223313
+01/07/2014,47.51903018
+01/08/2014,102.3992605
+01/09/2014,102.5496702
+01/10/2014,102.3533384
+01/11/2014,101.8451732
+01/12/2014,102.8815443
diff --git a/test/test_data/aggregate/aggregate_weight_shares.csv b/test/test_data/aggregate/aggregate_weight_shares.csv
@@ -0,0 +1,4 @@
+period,weights1,weights2,weights3
+01/02/2012,0.489537029,0.213620078,0.296842892
+01/02/2013,0.535477885,0.147572705,0.316949410
+01/02/2014,0.512055362,0.194043900,0.293900738
diff --git a/test/test_data/aggregate/aggregate_weights.csv b/test/test_data/aggregate/aggregate_weights.csv
@@ -0,0 +1,4 @@
+period,weights1,weights2,weights3
+01/02/2012,5.186964384,2.263444179,3.145244219
+01/02/2013,6.74500585,1.858860633,3.992369584
+01/02/2014,6.23115844,2.361303832,3.576453249
diff --git a/test/test_data/aggregate/aggregate_weights_hierarchy.csv b/test/test_data/aggregate/aggregate_weights_hierarchy.csv
@@ -0,0 +1,5 @@
+,top,top,top,top,top,top,top
+,cheese,cheese,cheese,beer,beer,wine,wine
+,A,B,C,D,E,F,G
+2017-02-01,12,2,11,5,15,19,13
+2018-02-01,11,3,13,5,16,21,15
diff --git a/test/test_data/aggregate/aggregate_weights_hierarchy_long.csv b/test/test_data/aggregate/aggregate_weights_hierarchy_long.csv
@@ -0,0 +1,9 @@
+,top,top,top,top,top,top,top
+,cheese,cheese,cheese,beer,beer,wine,wine
+,A,B,C,D,E,F,G
+2017-01-01,12.0,2.0,11.0,5.0,15.0,19.0,13.0
+2017-02-01,12.0,2.0,11.0,5.0,15.0,19.0,13.0
+2017-03-01,12.0,2.0,11.0,5.0,15.0,19.0,13.0
+2018-01-01,11.0,3.0,13.0,5.0,16.0,21.0,15.0
+2018-02-01,11.0,3.0,13.0,5.0,16.0,21.0,15.0
+2018-03-01,11.0,3.0,13.0,5.0,16.0,21.0,15.0
diff --git a/test/test_data/aggregate/aggregate_weights_missing.csv b/test/test_data/aggregate/aggregate_weights_missing.csv
@@ -0,0 +1,2 @@
+period,quote1,quote2,quote3
+01/01/2013,13.1223919,7.90254844,3.20531341
diff --git a/test/test_data/precon_test_data_calculator.xlsx b/test/test_data/precon_test_data_calculator.xlsx
diff --git a/test/test_data/weights/reindex_to_update_periods_weights.csv b/test/test_data/weights/reindex_to_update_periods_weights.csv
@@ -0,0 +1,37 @@
+period,weights1,weights2,weights3
+01/01/2016,5.186964384,2.263444179,3.145244219
+01/02/2016,5.186964384,2.263444179,3.145244219
+01/03/2016,5.186964384,2.263444179,3.145244219
+01/04/2016,5.186964384,2.263444179,3.145244219
+01/05/2016,5.186964384,2.263444179,3.145244219
+01/06/2016,5.186964384,2.263444179,3.145244219
+01/07/2016,5.186964384,2.263444179,3.145244219
+01/08/2016,5.186964384,2.263444179,3.145244219
+01/09/2016,5.186964384,2.263444179,3.145244219
+01/10/2016,5.186964384,2.263444179,3.145244219
+01/11/2016,5.186964384,2.263444179,3.145244219
+01/12/2016,5.186964384,2.263444179,3.145244219
+01/01/2017,6.74500585,1.858860633,3.992369584
+01/02/2017,6.74500585,1.858860633,3.992369584
+01/03/2017,6.74500585,1.858860633,3.992369584
+01/04/2017,6.74500585,1.858860633,3.992369584
+01/05/2017,6.74500585,1.858860633,3.992369584
+01/06/2017,6.74500585,1.858860633,3.992369584
+01/07/2017,6.74500585,1.858860633,3.992369584
+01/08/2017,6.74500585,1.858860633,3.992369584
+01/09/2017,6.74500585,1.858860633,3.992369584
+01/10/2017,6.74500585,1.858860633,3.992369584
+01/11/2017,6.74500585,1.858860633,3.992369584
+01/12/2017,6.74500585,1.858860633,3.992369584
+01/01/2018,6.23115844,2.361303832,3.576453249
+01/02/2018,6.23115844,2.361303832,3.576453249
+01/03/2018,6.23115844,2.361303832,3.576453249
+01/04/2018,6.23115844,2.361303832,3.576453249
+01/05/2018,6.23115844,2.361303832,3.576453249
+01/06/2018,6.23115844,2.361303832,3.576453249
+01/07/2018,6.23115844,2.361303832,3.576453249
+01/08/2018,6.23115844,2.361303832,3.576453249
+01/09/2018,6.23115844,2.361303832,3.576453249
+01/10/2018,6.23115844,2.361303832,3.576453249
+01/11/2018,6.23115844,2.361303832,3.576453249
+01/12/2018,6.23115844,2.361303832,3.576453249
diff --git a/test/test_data/weights/reindex_weights_weights.csv b/test/test_data/weights/reindex_weights_weights.csv
@@ -0,0 +1,37 @@
+period,weights1,weights2,weights3
+01/01/2012,5.186964384,2.263444179,3.145244219
+01/02/2012,5.186964384,2.263444179,3.145244219
+01/03/2012,5.186964384,2.263444179,3.145244219
+01/04/2012,5.186964384,2.263444179,3.145244219
+01/05/2012,5.186964384,2.263444179,3.145244219
+01/06/2012,5.186964384,2.263444179,3.145244219
+01/07/2012,5.186964384,2.263444179,3.145244219
+01/08/2012,5.186964384,2.263444179,3.145244219
+01/09/2012,5.186964384,2.263444179,3.145244219
+01/10/2012,5.186964384,2.263444179,3.145244219
+01/11/2012,5.186964384,2.263444179,3.145244219
+01/12/2012,5.186964384,2.263444179,3.145244219
+01/01/2013,6.745005850,1.858860633,3.992369584
+01/02/2013,6.745005850,1.858860633,3.992369584
+01/03/2013,6.745005850,1.858860633,3.992369584
+01/04/2013,6.745005850,1.858860633,3.992369584
+01/05/2013,6.745005850,1.858860633,3.992369584
+01/06/2013,6.745005850,1.858860633,3.992369584
+01/07/2013,6.745005850,1.858860633,3.992369584
+01/08/2013,6.745005850,1.858860633,3.992369584
+01/09/2013,6.745005850,1.858860633,3.992369584
+01/10/2013,6.745005850,1.858860633,3.992369584
+01/11/2013,6.745005850,1.858860633,3.992369584
+01/12/2013,6.745005850,1.858860633,3.992369584
+01/01/2014,6.231158440,2.361303832,3.576453249
+01/02/2014,6.231158440,2.361303832,3.576453249
+01/03/2014,6.231158440,2.361303832,3.576453249
+01/04/2014,6.231158440,2.361303832,3.576453249
+01/05/2014,6.231158440,2.361303832,3.576453249
+01/06/2014,6.231158440,2.361303832,3.576453249
+01/07/2014,6.231158440,2.361303832,3.576453249
+01/08/2014,6.231158440,2.361303832,3.576453249
+01/09/2014,6.231158440,2.361303832,3.576453249
+01/10/2014,6.231158440,2.361303832,3.576453249
+01/11/2014,6.231158440,2.361303832,3.576453249
+01/12/2014,6.231158440,2.361303832,3.576453249
diff --git a/test/test_helpers/csv_to_text.py b/test/test_helpers/csv_to_text.py
@@ -0,0 +1,86 @@
+"""Helper script to generate tuple txt data from CSVs."""
+import os
+import glob
+from pathlib import Path
+from typing import Sequence, Callable
+
+import pandas as pd
+from pandas._typing import TimestampConvertibleTypes
+
+# Define the directory to find CSVs, and optional output TXT directory.
+CSV_DIR = '/home/cdsw/tests/local_collection/test_data/unit_tests'
+TXT_DIR = CSV_DIR
+
+
+def main() -> None:
+    """Convert the files."""
+    # If you only want specific files in the CSV directory list them
+    # here.
+    # files = [
+    #     'myfile1.csv',
+    #     'myfil2.csv',
+    # ]
+    # convert_all_csvs(files)
+
+    convert_all_csvs()
+
+
+def _generate_test_data(df: pd.DataFrame, filename: str) -> None:
+    """Generate hard-coded test data from a DataFrame."""
+    headers = (df.index.name,) + tuple(df.columns)
+    df_as_list = list(df.itertuples(index=True, name=None))
+
+    with open(filename, 'w') as f:
+        f.write(str(headers)+',\n')
+        f.write('\n'.join(str(t)+',' for t in df_as_list))
+
+
+def _load(
+    filename: str,
+    date_parser: Callable[[TimestampConvertibleTypes], pd.Series] = None,
+) -> pd.DataFrame:
+    """Quick load file with optional date parser."""
+    return pd.read_csv(
+        filename,
+        index_col=0,
+        parse_dates=True,
+        dayfirst=True,
+        date_parser=date_parser,
+    )
+
+
+def convert_all_csvs(
+    files: Sequence[str] = None,
+    date_parser: Callable[[TimestampConvertibleTypes], pd.Series] = None,
+) -> None:
+    """Convert all CSVs to tuple text data.
+
+    Default behaviour is to convert all files in directory given by
+    CSV_TXT, but files can be specified using files argument.
+
+    Parameters
+    ----------
+    files: list of str, optional
+        The filenames to convert.
+    date_parser: callable, optional
+        To convert date types in formats not picked up by pandas i.e.
+        "201701".
+
+    """
+    if not files:
+        files = glob.glob(CSV_DIR + '*.csv')
+    else:
+        files = [os.path.join(CSV_DIR, file) for file in files]
+
+    txt_files = [
+        os.path.join(TXT_DIR, Path(s).name.split('.')[0] + '.txt')
+        for s in files
+    ]
+
+    for i, file in enumerate(files):
+        df = _load(file, date_parser)
+        _generate_test_data(df, txt_files[i])
+
+
+if __name__ == "__main__":
+    main()
diff --git a/test/test_helpers/generate_index_data.py b/test/test_helpers/generate_index_data.py
