Update pre-commit and fix linter issues

Author: mkstratos

.pre-commit-config.yaml

@@ -1,21 +1,24 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
       - id: check-toml
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
+      - id: check-yaml
+        args: [--unsafe]
+      - id: debug-statements
+      - id: destroyed-symlinks
+      - id: detect-private-key
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.3.7
+    rev: v0.13.0
     hooks:
       # Run the linter.
-      - id: ruff
+      - id: ruff-check
+      # Run the formatter.
+      - id: ruff-format
+  - repo: https://github.com/pycqa/isort
+    rev: 6.0.1
+    hooks:
+      - id: isort

detclim/bootstrap_tests.py

@@ -101,17 +101,13 @@ def mannwhitney(data_1, data_2):
 def wilcoxon(data_1, data_2):
     """Perform a Wiloxon Signed Rank Test, return P-value."""
     with np.errstate(divide="ignore", invalid="ignore"):
-        return sts.wilcoxon(
-            data_1 - data_2, axis=1
-        ).pvalue  # pyright: ignore[reportAttributeAccessIssue]
+        return sts.wilcoxon(data_1 - data_2, axis=1).pvalue  # pyright: ignore[reportAttributeAccessIssue]
 
 
 def epps_singleton(data_1, data_2):
     """Perform a 2 sample Epps Singleton test, return P-value."""
     try:
-        _out = sts.epps_singleton_2samp(
-            data_1, data_2, axis=1
-        ).pvalue  # pyright: ignore[reportCallIssue]
+        _out = sts.epps_singleton_2samp(data_1, data_2, axis=1).pvalue  # pyright: ignore[reportCallIssue]
     except np.linalg.LinAlgError:
         _out = np.ones(data_1.shape[0])
     return _out

detclim/notebooks/BootstrapTest.ipynb

@@ -18,8 +18,7 @@
     "from functools import partial\n",
     "from statsmodels.stats import multitest as smm\n",
     "import multiprocessing as mp\n",
-    "import seaborn as sns\n",
-    "from dask.distributed import Client"
+    "import seaborn as sns"
    ]
   },
   {
@@ -39,13 +38,12 @@
     "    ens_idx = sorted(range(ens_min, ens_max + 1))\n",
     "    assert len(ens_idx) > ens_size, \"ENSEMBLE SIZE MUST BE SMALLER THAN ENSEMBLE RANGE\"\n",
     "    if not with_repl and not uniq:\n",
-    "        selected = [\n",
-    "            random.sample(ens_idx, ens_size)\n",
-    "            for _ in range(ncases)\n",
-    "        ]\n",
+    "        selected = [random.sample(ens_idx, ens_size) for _ in range(ncases)]\n",
     "    elif not with_repl:\n",
     "        _sel = random.sample(ens_idx, ens_size * ncases)\n",
-    "        selected = [_sel[idx * ens_size : (idx + 1) * ens_size] for idx in range(ncases)]\n",
+    "        selected = [\n",
+    "            _sel[idx * ens_size : (idx + 1) * ens_size] for idx in range(ncases)\n",
+    "        ]\n",
     "    else:\n",
     "        selected = [\n",
     "            [random.randint(ens_min, ens_max) for _ in range(ens_size)]\n",
@@ -63,6 +61,7 @@
     "    _res = sts.mstats.ks_2samp(data_x, data_y)\n",
     "    return _res[1]\n",
     "\n",
+    "\n",
     "def cvm_2samp(data_x, data_y):\n",
     "    \"\"\"Perform a 2 sample Cramer von Mises test, map output to a tuple.\"\"\"\n",
     "    _res = sts.cramervonmises_2samp(data_x, data_y)\n",
@@ -71,9 +70,11 @@
     "\n",
     "def anderson_pval(data_1, data_2):\n",
     "    try:\n",
-    "        _res = sts.anderson_ksamp([data_1, data_2], method=sts.PermutationMethod(n_resamples=100))\n",
+    "        _res = sts.anderson_ksamp(\n",
+    "            [data_1, data_2], method=sts.PermutationMethod(n_resamples=100)\n",
+    "        )\n",
     "    except ValueError:\n",
-    "        return 1.\n",
+    "        return 1.0\n",
     "    return _res.pvalue\n",
     "\n",
     "\n",
@@ -87,7 +88,6 @@
     "\n",
     "\n",
     "def epps_singleton(data_1, data_2):\n",
-    "\n",
     "    # print(data_1.shape, data_2.shape)\n",
     "    try:\n",
     "        _out = sts.epps_singleton_2samp(data_1, data_2, axis=1).pvalue\n",
@@ -97,7 +97,6 @@
     "\n",
     "\n",
     "def test_all_times(data, ens_ids, test_fcn):\n",
-    "\n",
     "    \"\"\"Perform statistical test on two arrays across all times in the array.\n",
     "\n",
     "    Parameters\n",
@@ -117,9 +116,7 @@
     "\n",
     "    _pval = test_fcn(data_1.T, data_2.T)\n",
     "    try:\n",
-    "        _out = xr.DataArray(\n",
-    "            data=_pval, dims=(\"time\",), coords={\"time\": data.time}\n",
-    "        )\n",
+    "        _out = xr.DataArray(data=_pval, dims=(\"time\",), coords={\"time\": data.time})\n",
     "    except ValueError as _err:\n",
     "        print(_err)\n",
     "        return None\n",
@@ -178,9 +175,7 @@
     "\n",
     "# _ds_all = xr.concat([_ds_ctl, _ds_exp], dim=\"exp\")\n",
     "_ds_all = xr.concat([_ds_ctl, _ds_exp], dim=\"exp\")\n",
-    "dvars = json.loads(\n",
-    "    open(\"../new_vars.json\", \"r\", encoding=\"utf-8\").read()\n",
-    ")[\"default\"]\n",
+    "dvars = json.loads(open(\"../new_vars.json\", \"r\", encoding=\"utf-8\").read())[\"default\"]\n",
     "\n",
     "_ds_all_mean = _ds_all[dvars].map(rolling_mean_data, period_len=12)\n",
     "_emin = _ds_all_mean.ens.values.min()\n",
@@ -213,7 +208,10 @@
     "    unique = True\n",
     "else:\n",
     "    unique = False\n",
-    "ens_sel = [randomise_new(_emin, _emax, ens_size=ens_size, ncases=2, uniq=unique) for _ in range(ninst)]"
+    "ens_sel = [\n",
+    "    randomise_new(_emin, _emax, ens_size=ens_size, ncases=2, uniq=unique)\n",
+    "    for _ in range(ninst)\n",
+    "]"
    ]
   },
   {
@@ -225,7 +223,9 @@
    "source": [
     "%%time\n",
     "# ks_bootsrap_part = partial(ks_bootstrap, data=_ds_all_mean[dvars])\n",
-    "ks_bootstrap_part = partial(bootstrap_test, data=_ds_all_mean[dvars], test_fcn=ks_test_vec)\n",
+    "ks_bootstrap_part = partial(\n",
+    "    bootstrap_test, data=_ds_all_mean[dvars], test_fcn=ks_test_vec\n",
+    ")\n",
     "with mp.Pool(16) as pool:\n",
     "    pvals_out_ks = xr.concat(pool.map(ks_bootstrap_part, ens_sel), dim=\"iter\")"
    ]
@@ -238,7 +238,9 @@
    "outputs": [],
    "source": [
     "%%time\n",
-    "es_bootstrap_part = partial(bootstrap_test, data=_ds_all_mean[dvars], test_fcn=epps_singleton)\n",
+    "es_bootstrap_part = partial(\n",
+    "    bootstrap_test, data=_ds_all_mean[dvars], test_fcn=epps_singleton\n",
+    ")\n",
     "with mp.Pool(16) as pool:\n",
     "    pvals_out_es = xr.concat(pool.map(es_bootstrap_part, ens_sel), dim=\"iter\")"
    ]
@@ -251,9 +253,11 @@
    "outputs": [],
    "source": [
     "%%time\n",
-    "mw_bootstrap_part = partial(bootstrap_test, data=_ds_all_mean[dvars], test_fcn=mannwhitney)\n",
+    "mw_bootstrap_part = partial(\n",
+    "    bootstrap_test, data=_ds_all_mean[dvars], test_fcn=mannwhitney\n",
+    ")\n",
     "with mp.Pool(16) as pool:\n",
-    "    pvals_out_mw = xr.concat(pool.map(mw_bootstrap_part, ens_sel), dim=\"iter\")\n"
+    "    pvals_out_mw = xr.concat(pool.map(mw_bootstrap_part, ens_sel), dim=\"iter\")"
    ]
   },
   {
@@ -275,7 +279,9 @@
    "outputs": [],
    "source": [
     "%%time\n",
-    "cvm_bootstrap_part = partial(bootstrap_test, data=_ds_all_mean[dvars], test_fcn=cvm_test_vec)\n",
+    "cvm_bootstrap_part = partial(\n",
+    "    bootstrap_test, data=_ds_all_mean[dvars], test_fcn=cvm_test_vec\n",
+    ")\n",
     "with mp.Pool(16) as pool:\n",
     "    pvals_out_cvm = xr.concat(pool.map(cvm_bootstrap_part, ens_sel), dim=\"iter\")"
    ]
@@ -363,7 +369,17 @@
     "    # _ = axis[idx].semilogy(pvals, color=\"grey\", lw=0.5)\n",
     "    _ = axis[idx].semilogy(np.median(pvals, axis=1), color=\"k\")\n",
     "    # methods = [\"fdr_bh\", \"fdr_by\", \"bonferroni\", \"sidak\", \"holm-sidak\", \"holm\", \"simes-hochberg\", \"hommel\", \"fdr_tsbh\", \"fdr_tsbky\"]\n",
-    "    methods = [\"fdr_bh\", \"fdr_by\", \"bonferroni\", \"sidak\", \"holm-sidak\", \"simes-hochberg\", \"hommel\", \"fdr_tsbh\", \"fdr_tsbky\"]\n",
+    "    methods = [\n",
+    "        \"fdr_bh\",\n",
+    "        \"fdr_by\",\n",
+    "        \"bonferroni\",\n",
+    "        \"sidak\",\n",
+    "        \"holm-sidak\",\n",
+    "        \"simes-hochberg\",\n",
+    "        \"hommel\",\n",
+    "        \"fdr_tsbh\",\n",
+    "        \"fdr_tsbky\",\n",
+    "    ]\n",
     "    # methods = [\"fdr_bh\"]\n",
     "    for _method in methods:\n",
     "        _pval_cr = np.array(\n",
@@ -392,7 +408,10 @@
    "outputs": [],
    "source": [
     "nreject = {\n",
-    "    mode: [(pvals_all[mode][i, :, -1] < 0.05).sum() for i in range(pvals_all[mode].shape[0])]\n",
+    "    mode: [\n",
+    "        (pvals_all[mode][i, :, -1] < 0.05).sum()\n",
+    "        for i in range(pvals_all[mode].shape[0])\n",
+    "    ]\n",
     "    for mode in pvals_all\n",
     "}\n",
     "nreject_cr = {}\n",
@@ -411,7 +430,9 @@
     "            for kdx in range(pvals_all[mode].shape[1])\n",
     "        ]\n",
     "    )\n",
-    "    nreject_cr[mode] = [(_pval_cr[:, i] < 0.05).sum() for i in range(pvals_all[mode].shape[0])]"
+    "    nreject_cr[mode] = [\n",
+    "        (_pval_cr[:, i] < 0.05).sum() for i in range(pvals_all[mode].shape[0])\n",
+    "    ]"
    ]
   },
   {
@@ -466,7 +487,7 @@
     "    # plt.gca().set_xlim([26, 46])\n",
     "    # plt.gca().set_ylim([0, 40])\n",
     "    plt.title(mode)\n",
-    "plt.tight_layout()\n"
+    "plt.tight_layout()"
    ]
   },
   {
@@ -517,7 +538,7 @@
     "            \"desc\": f\"2-sample {test_name} p-value\",\n",
     "            \"long_name\": f\"{test_name}_pvalue\",\n",
     "            \"short_name\": f\"{test_name}_pvalue\",\n",
-    "            },\n",
+    "        },\n",
     "    )"
    ]
   },
@@ -530,7 +551,9 @@
    "source": [
     "ds_out = {}\n",
     "for _test in pvals_all:\n",
-    "    ds_out[f\"{_test}_pval\"] = to_dataarray(pvals_all[_test], dvars, pvals_out_ks.time, _test)\n",
+    "    ds_out[f\"{_test}_pval\"] = to_dataarray(\n",
+    "        pvals_all[_test], dvars, pvals_out_ks.time, _test\n",
+    "    )\n",
     "xr.Dataset(ds_out)"
    ]
   },
@@ -541,7 +564,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "plt.loglog(pvals_all[\"ks\"][:, :, -1].flatten(), pvals_all[\"cvm\"][:, :, -1].flatten(), \".\", alpha=0.5)"
+    "plt.loglog(\n",
+    "    pvals_all[\"ks\"][:, :, -1].flatten(),\n",
+    "    pvals_all[\"cvm\"][:, :, -1].flatten(),\n",
+    "    \".\",\n",
+    "    alpha=0.5,\n",
+    ")"
    ]
   },
   {
@@ -552,7 +580,9 @@
    "outputs": [],
    "source": [
     "_ds_all\n",
-    "mwu = sts.mannwhitneyu(_ds_all[\"T\"].isel(exp=0).values, _ds_all[\"T\"].isel(exp=1).values, axis=0)"
+    "mwu = sts.mannwhitneyu(\n",
+    "    _ds_all[\"T\"].isel(exp=0).values, _ds_all[\"T\"].isel(exp=1).values, axis=0\n",
+    ")"
    ]
   },
   {
@@ -562,7 +592,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "esp = sts.epps_singleton_2samp(_ds_all[\"T\"].isel(exp=0).values, _ds_all[\"T\"].isel(exp=1).values, axis=0).pvalue"
+    "esp = sts.epps_singleton_2samp(\n",
+    "    _ds_all[\"T\"].isel(exp=0).values, _ds_all[\"T\"].isel(exp=1).values, axis=0\n",
+    ").pvalue"
    ]
   },
   {