diff --git a/demo/control/control_app.py b/demo/control/control_app.py new file mode 100644 index 00000000..39d9c169 --- /dev/null +++ b/demo/control/control_app.py @@ -0,0 +1,108 @@ +# --- +# jupyter: +# jupytext: +# text_representation: +# extension: .py +# format_name: light +# format_version: '1.5' +# kernelspec: +# display_name: demo-ssb-dash +# language: python +# name: demo-ssb-dash +# --- + +# + +import os # Nødvendig for oppsett + +from control_code import DemoControls # For å kunne bruke kontroll-modulen + +# Moduler importert fra biblioteket +# from ssb_dash_framework import HBMethod +from ssb_dash_framework import AggDistPlotterWindow +from ssb_dash_framework import AltinnControlViewWindow +from ssb_dash_framework import AltinnSkjemadataEditor +from ssb_dash_framework import FreeSearchTab +from ssb_dash_framework import app_setup +from ssb_dash_framework import main_layout +from ssb_dash_framework import set_eimerdb_connection +from ssb_dash_framework import set_variables + +# from egentilpassing.hb_method import hb_get_data + + +# Kobling til EimerDB som ligger i fellesbøtta. +set_eimerdb_connection( + "ssb-dapla-felles-data-produkt-prod", + "produksjonstilskudd_altinn3", +) +selected_time_units = ["aar"] # Tidsenheten(e) til dataene + +# Basic app-oppsett +port = 8070 +service_prefix = os.getenv("JUPYTERHUB_SERVICE_PREFIX", "/") +domain = os.getenv("JUPYTERHUB_HTTP_REFERER", None) +app = app_setup(port, service_prefix, "lumen", logging_level="debug", log_to_file=True) + +# VARIABLER: Velg hvilke variabler appen skal bruke. Default values er valgfritt, men anbefalt. +set_variables( + [ + "aar", + "ident", + "statistikkvariabel", + "altinnskjema", + "valgt_tabell", + "refnr", + ] +) +default_values = { + "aar": "2023", + "statistikkvariabel": "fulldyrket", + "valgt_tabell": "skjemadata_hoved", + "altinnskjema": "RA-7357", +} + +# Uncomment this if you have access to the VoF shared bucket. +# bof_module = BofInformationTab( +# variableselector_foretak_name="ident" +# ) + +# Legg in alle tabene i denne lista +tab_list = [ + AltinnSkjemadataEditor(time_units=selected_time_units, variable_connection={}), + FreeSearchTab(), + # bof_module # Uncomment this if you have access to the VoF shared bucket. +] + +# MODALENE: Lag en liste med alle modalene +# hb = HBMethod( +# database=conn, +# hb_get_data_func=hb_get_data, +# selected_state_keys=["aar", "statistikkvariabel"], +# selected_ident="ident", +# variable="verdi", +# ) + +# visualiseringsbyggermodul = VisualizationBuilder(conn) +# datafangstmodalen = AltinnDataCaptureWindow( +# time_units=selected_time_units +# ) +aggdistplotter = AggDistPlotterWindow(selected_time_units) + +controls = AltinnControlViewWindow( + time_units=["aar"], control_dict={"RA-7357": DemoControls} +) + +# MODALENE: Lag en liste med alle modalene +modal_list = [ + # datafangstmodalen, + aggdistplotter, + controls, + # hb, + # visualiseringsbyggermodul, +] + +# Denne linja genererer layouten til hele appen. Her legges listene med valgt innhold for modalene, fanene og variablene. +app.layout = main_layout(modal_list, tab_list, default_values=default_values) + +if __name__ == "__main__": + app.run(debug=True, port=port, jupyter_server_url=domain, jupyter_mode="tab") diff --git a/demo/control/control_code.py b/demo/control/control_code.py new file mode 100644 index 00000000..11b9a6bd --- /dev/null +++ b/demo/control/control_code.py @@ -0,0 +1,119 @@ +# --- +# jupyter: +# jupytext: +# cell_metadata_filter: -papermill,tags +# custom_cell_magics: kql +# text_representation: +# extension: .py +# format_name: percent +# format_version: '1.3' +# jupytext_version: 1.16.2 +# kernelspec: +# display_name: .venv +# language: python +# name: python3 +# --- + + +import eimerdb as db +import pandas as pd + +# %% metadata={} +from ssb_dash_framework import ControlFrameworkBase +from ssb_dash_framework import register_control + +conn = db.EimerDBInstance( + "ssb-dapla-felles-data-produkt-prod", + "produksjonstilskudd_altinn3", +) + + +class DemoControls(ControlFrameworkBase): + def __init__( + self, + time_units: list[int | str], + applies_to_subset: dict[str, int | str], + ) -> None: + super().__init__(time_units, applies_to_subset) + + @register_control( + kontrollid="nye", + kontrolltype="I", + beskrivelse="Nye enheter for året", + kontrollerte_variabler=["ident"], + sorteringsvariabel="fulldyrket", + sortering="ASC", + ) + def control_nye(self): + aar = int(self.applies_to_subset["aar"][0]) + + df = conn.query("SELECT * FROM skjemamottak") + + enheter_fjoraar = list(df.loc[df["aar"] == aar - 1]["ident"].unique()) + + nye_enheter = df.loc[(df["aar"] == aar) & (~df["ident"].isin(enheter_fjoraar))] + ikke_nye_enheter = df.loc[ + (df["aar"] == aar) & (df["ident"].isin(enheter_fjoraar)) + ] + + nye_enheter["utslag"] = True + ikke_nye_enheter["utslag"] = False + + df = pd.concat([nye_enheter, ikke_nye_enheter]) + df["verdi"] = 0 + + return df[["aar", "skjema", "ident", "refnr", "utslag", "verdi"]] + + @register_control( + kontrollid="diff", + kontrolltype="S", + beskrivelse="Stor differanse mot fjoråret", + kontrollerte_variabler=["fulldyrket"], + sorteringsvariabel="fulldyrket", + sortering="ASC", + ) + def control_diff(self): + aar = int(self.applies_to_subset["aar"][0]) + df = conn.query("SELECT * FROM skjemadata_hoved") + df = df.loc[df["variabel"] == "fulldyrket"] + diff_df = df.loc[(df["aar"].isin([aar, aar - 1]))] + diff_df = diff_df.pivot( + index=["ident", "variabel"], columns="aar", values="verdi" + ).reset_index() + diff_df[aar - 1] = diff_df[aar - 1].astype(float) + diff_df[aar] = diff_df[aar].astype(float) + diff_df = diff_df.loc[(diff_df[aar - 1] > 0) & (diff_df[aar] > 0)] + + diff_df["differanse"] = diff_df[aar] - diff_df[aar - 1] + diff_df["prosent_endring"] = (diff_df["differanse"] / diff_df[aar - 1]) * 100 + + diff_df["utslag"] = False + diff_df.loc[(abs(diff_df["prosent_endring"]) > 100), "utslag"] = True + + diff_df["aar"] = aar + diff_df["skjema"] = "RA-7357" + diff_df["kontrollid"] = "diff" + diff_df["verdi"] = diff_df["prosent_endring"].astype(int) + + diff_df = diff_df.merge( + df.loc[df["aar"] == aar][["ident", "refnr"]], on="ident" + ) + return diff_df[ + ["aar", "skjema", "ident", "refnr", "kontrollid", "utslag", "verdi"] + ] + + +if __name__ == "__main__": + + test = DemoControls( + time_units=["aar"], + applies_to_subset={"aar": 2024, "skjema": "RA-7357"}, + conn=conn, + ) + + test.register_all_controls() + test.execute_controls() + + print(conn.query("SELECT * FROM kontroller")) + + print(conn.query("SELECT * FROM kontrollutslag")) diff --git a/poetry.lock b/poetry.lock index aeb4cb23..b8dd40ac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -1571,7 +1571,7 @@ grpcio-status = [ ] proto-plus = [ {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, - {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.22.3,<2.0.0"}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" requests = ">=2.18.0,<3.0.0" @@ -1702,7 +1702,7 @@ grpcio = [ ] proto-plus = [ {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, - {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, ] protobuf = ">=3.20.2,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" @@ -4048,105 +4048,47 @@ test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "setuptools"] [[package]] name = "psycopg" -version = "3.3.2" +version = "3.3.3" description = "PostgreSQL database adapter for Python" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "psycopg-3.3.2-py3-none-any.whl", hash = "sha256:3e94bc5f4690247d734599af56e51bae8e0db8e4311ea413f801fef82b14a99b"}, - {file = "psycopg-3.3.2.tar.gz", hash = "sha256:707a67975ee214d200511177a6a80e56e654754c9afca06a7194ea6bbfde9ca7"}, + {file = "psycopg-3.3.3-py3-none-any.whl", hash = "sha256:f96525a72bcfade6584ab17e89de415ff360748c766f0106959144dcbb38c698"}, + {file = "psycopg-3.3.3.tar.gz", hash = "sha256:5e9a47458b3c1583326513b2556a2a9473a1001a56c9efe9e587245b43148dd9"}, ] [package.dependencies] +psycopg-pool = {version = "*", optional = true, markers = "extra == \"pool\""} typing-extensions = {version = ">=4.6", markers = "python_version < \"3.13\""} tzdata = {version = "*", markers = "sys_platform == \"win32\""} [package.extras] -binary = ["psycopg-binary (==3.3.2) ; implementation_name != \"pypy\""] -c = ["psycopg-c (==3.3.2) ; implementation_name != \"pypy\""] -dev = ["ast-comments (>=1.1.2)", "black (>=24.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] +binary = ["psycopg-binary (==3.3.3) ; implementation_name != \"pypy\""] +c = ["psycopg-c (==3.3.3) ; implementation_name != \"pypy\""] +dev = ["ast-comments (>=1.1.2)", "black (>=26.1.0)", "codespell (>=2.2)", "cython-lint (>=0.16)", "dnspython (>=2.1)", "flake8 (>=4.0)", "isort-psycopg", "isort[colors] (>=6.0)", "mypy (>=1.19.0)", "pre-commit (>=4.0.1)", "types-setuptools (>=57.4)", "types-shapely (>=2.0)", "wheel (>=0.37)"] docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] pool = ["psycopg-pool"] test = ["anyio (>=4.0)", "mypy (>=1.19.0) ; implementation_name != \"pypy\"", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] [[package]] -name = "psycopg2-binary" -version = "2.9.11" -description = "psycopg2 - Python-PostgreSQL Database Adapter" +name = "psycopg-pool" +version = "3.3.0" +description = "Connection Pool for Psycopg" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] files = [ - {file = "psycopg2-binary-2.9.11.tar.gz", hash = "sha256:b6aed9e096bf63f9e75edf2581aa9a7e7186d97ab5c177aa6c87797cd591236c"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6fe6b47d0b42ce1c9f1fa3e35bb365011ca22e39db37074458f27921dca40f2"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a6c0e4262e089516603a09474ee13eabf09cb65c332277e39af68f6233911087"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c47676e5b485393f069b4d7a811267d3168ce46f988fa602658b8bb901e9e64d"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a28d8c01a7b27a1e3265b11250ba7557e5f72b5ee9e5f3a2fa8d2949c29bf5d2"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f3f2732cf504a1aa9e9609d02f79bea1067d99edf844ab92c247bbca143303b"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:865f9945ed1b3950d968ec4690ce68c55019d79e4497366d36e090327ce7db14"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91537a8df2bde69b1c1db01d6d944c831ca793952e4f57892600e96cee95f2cd"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4dca1f356a67ecb68c81a7bc7809f1569ad9e152ce7fd02c2f2036862ca9f66b"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0da4de5c1ac69d94ed4364b6cbe7190c1a70d325f112ba783d83f8440285f152"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37d8412565a7267f7d79e29ab66876e55cb5e8e7b3bbf94f8206f6795f8f7e7e"}, - {file = "psycopg2_binary-2.9.11-cp310-cp310-win_amd64.whl", hash = "sha256:c665f01ec8ab273a61c62beeb8cce3014c214429ced8a308ca1fc410ecac3a39"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0e8480afd62362d0a6a27dd09e4ca2def6fa50ed3a4e7c09165266106b2ffa10"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:763c93ef1df3da6d1a90f86ea7f3f806dc06b21c198fa87c3c25504abec9404a"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2e164359396576a3cc701ba8af4751ae68a07235d7a380c631184a611220d9a4"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:d57c9c387660b8893093459738b6abddbb30a7eab058b77b0d0d1c7d521ddfd7"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c226ef95eb2250974bf6fa7a842082b31f68385c4f3268370e3f3870e7859ee"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a311f1edc9967723d3511ea7d2708e2c3592e3405677bf53d5c7246753591fbb"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ebb415404821b6d1c47353ebe9c8645967a5235e6d88f914147e7fd411419e6f"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f07c9c4a5093258a03b28fab9b4f151aa376989e7f35f855088234e656ee6a94"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:00ce1830d971f43b667abe4a56e42c1e2d594b32da4802e44a73bacacb25535f"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:cffe9d7697ae7456649617e8bb8d7a45afb71cd13f7ab22af3e5c61f04840908"}, - {file = "psycopg2_binary-2.9.11-cp311-cp311-win_amd64.whl", hash = "sha256:304fd7b7f97eef30e91b8f7e720b3db75fee010b520e434ea35ed1ff22501d03"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be9b840ac0525a283a96b556616f5b4820e0526addb8dcf6525a0fa162730be4"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f090b7ddd13ca842ebfe301cd587a76a4cf0913b1e429eb92c1be5dbeb1a19bc"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ab8905b5dcb05bf3fb22e0cf90e10f469563486ffb6a96569e51f897c750a76a"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf940cd7e7fec19181fdbc29d76911741153d51cab52e5c21165f3262125685e"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa0f693d3c68ae925966f0b14b8edda71696608039f4ed61b1fe9ffa468d16db"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a1cf393f1cdaf6a9b57c0a719a1068ba1069f022a59b8b1fe44b006745b59757"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7a6beb4beaa62f88592ccc65df20328029d721db309cb3250b0aae0fa146c3"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:31b32c457a6025e74d233957cc9736742ac5a6cb196c6b68499f6bb51390bd6a"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:edcb3aeb11cb4bf13a2af3c53a15b3d612edeb6409047ea0b5d6a21a9d744b34"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b6d93d7c0b61a1dd6197d208ab613eb7dcfdcca0a49c42ceb082257991de9d"}, - {file = "psycopg2_binary-2.9.11-cp312-cp312-win_amd64.whl", hash = "sha256:b33fabeb1fde21180479b2d4667e994de7bbf0eec22832ba5d9b5e4cf65b6c6d"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b8fb3db325435d34235b044b199e56cdf9ff41223a4b9752e8576465170bb38c"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:366df99e710a2acd90efed3764bb1e28df6c675d33a7fb40df9b7281694432ee"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c55b385daa2f92cb64b12ec4536c66954ac53654c7f15a203578da4e78105c0"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c0377174bf1dd416993d16edc15357f6eb17ac998244cca19bc67cdc0e2e5766"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6ff3335ce08c75afaed19e08699e8aacf95d4a260b495a4a8545244fe2ceb3"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84011ba3109e06ac412f95399b704d3d6950e386b7994475b231cf61eec2fc1f"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ba34475ceb08cccbdd98f6b46916917ae6eeb92b5ae111df10b544c3a4621dc4"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b31e90fdd0f968c2de3b26ab014314fe814225b6c324f770952f7d38abf17e3c"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d526864e0f67f74937a8fce859bd56c979f5e2ec57ca7c627f5f1071ef7fee60"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04195548662fa544626c8ea0f06561eb6203f1984ba5b4562764fbeb4c3d14b1"}, - {file = "psycopg2_binary-2.9.11-cp313-cp313-win_amd64.whl", hash = "sha256:efff12b432179443f54e230fdf60de1f6cc726b6c832db8701227d089310e8aa"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:92e3b669236327083a2e33ccfa0d320dd01b9803b3e14dd986a4fc54aa00f4e1"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e0deeb03da539fa3577fcb0b3f2554a97f7e5477c246098dbb18091a4a01c16f"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b52a3f9bb540a3e4ec0f6ba6d31339727b2950c9772850d6545b7eae0b9d7c5"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:db4fd476874ccfdbb630a54426964959e58da4c61c9feba73e6094d51303d7d8"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:47f212c1d3be608a12937cc131bd85502954398aaa1320cb4c14421a0ffccf4c"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e35b7abae2b0adab776add56111df1735ccc71406e56203515e228a8dc07089f"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fcf21be3ce5f5659daefd2b3b3b6e4727b028221ddc94e6c1523425579664747"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9bd81e64e8de111237737b29d68039b9c813bdf520156af36d26819c9a979e5f"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:32770a4d666fbdafab017086655bcddab791d7cb260a16679cc5a7338b64343b"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3cb3a676873d7506825221045bd70e0427c905b9c8ee8d6acd70cfcbd6e576d"}, - {file = "psycopg2_binary-2.9.11-cp314-cp314-win_amd64.whl", hash = "sha256:4012c9c954dfaccd28f94e84ab9f94e12df76b4afb22331b1f0d3154893a6316"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:20e7fb94e20b03dcc783f76c0865f9da39559dcc0c28dd1a3fce0d01902a6b9c"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4bdab48575b6f870f465b397c38f1b415520e9879fdf10a53ee4f49dcbdf8a21"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9d3a9edcfbe77a3ed4bc72836d466dfce4174beb79eda79ea155cc77237ed9e8"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:44fc5c2b8fa871ce7f0023f619f1349a0aa03a0857f2c96fbc01c657dcbbdb49"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9c55460033867b4622cda1b6872edf445809535144152e5d14941ef591980edf"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2d11098a83cca92deaeaed3d58cfd150d49b3b06ee0d0852be466bf87596899e"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:691c807d94aecfbc76a14e1408847d59ff5b5906a04a23e12a89007672b9e819"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:8b81627b691f29c4c30a8f322546ad039c40c328373b11dff7490a3e1b517855"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:b637d6d941209e8d96a072d7977238eea128046effbf37d1d8b2c0764750017d"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:41360b01c140c2a03d346cec3280cf8a71aa07d94f3b1509fa0161c366af66b4"}, - {file = "psycopg2_binary-2.9.11-cp39-cp39-win_amd64.whl", hash = "sha256:875039274f8a2361e5207857899706da840768e2a775bf8c65e82f60b197df02"}, + {file = "psycopg_pool-3.3.0-py3-none-any.whl", hash = "sha256:2e44329155c410b5e8666372db44276a8b1ebd8c90f1c3026ebba40d4bc81063"}, + {file = "psycopg_pool-3.3.0.tar.gz", hash = "sha256:fa115eb2860bd88fce1717d75611f41490dec6135efb619611142b24da3f6db5"}, ] +[package.dependencies] +typing-extensions = ">=4.6" + +[package.extras] +test = ["anyio (>=4.0)", "mypy (>=1.14)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -7020,4 +6962,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "0cd7a4eceaba36a91a4630af1c9ada66527626a663573fc1fd108026736a2dda" +content-hash = "1d8e5876c6baf6161b8b591d81834aec9f05f076d75778a6e31219edc8d9e041" diff --git a/pyproject.toml b/pyproject.toml index 4969349c..d018ffd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "mpmath>=1.3.0", # Pimemorizer uses this "orjson (>=3.10.16,<4.0.0)", "ipykernel (>=6.29.5,<7.0.0)", - "psycopg2-binary (>=2.9.10,<3.0.0)", "geopandas (>=1.1.1,<2.0.0)", "ssb-eimerdb (>=0.2.6,<0.3.0)", "sqlalchemy (>=2.0.43,<3.0.0)", @@ -27,6 +26,7 @@ dependencies = [ "dash-bootstrap-templates (>=2.1.0,<3.0.0)", "duckdb (==1.3.2)", "ssb-poc-statlog-model (>=1.0.0,<2.0.0)", + "psycopg[pool] (>=3.3.3,<4.0.0)", ] [project.urls] diff --git a/src/ssb_dash_framework/__init__.py b/src/ssb_dash_framework/__init__.py index 4139a8ba..16828d7c 100644 --- a/src/ssb_dash_framework/__init__.py +++ b/src/ssb_dash_framework/__init__.py @@ -39,11 +39,11 @@ from .modules import HBMethod from .modules import HBMethodWindow from .modules import MacroModule -from .modules import MacroModuleTab -from .modules import MacroModuleWindow from .modules import MacroModuleConsolidated from .modules import MacroModuleConsolidatedTab from .modules import MacroModuleConsolidatedWindow +from .modules import MacroModuleTab +from .modules import MacroModuleWindow from .modules import MapDisplay from .modules import MapDisplayTab from .modules import MapDisplayWindow @@ -76,6 +76,8 @@ from .utils import DemoDataCreator from .utils import TabImplementation from .utils import WindowImplementation +from .utils import _get_connection_callable +from .utils import _get_connection_object from .utils import _get_kostra_r from .utils import active_no_duplicates_refnr_list from .utils import conn_is_ibis @@ -83,9 +85,13 @@ from .utils import create_database from .utils import create_database_engine from .utils import enable_app_logging +from .utils import get_connection from .utils import hb_method from .utils import ibis_filter_with_dict from .utils import module_validator +from .utils import set_connection +from .utils import set_eimerdb_connection +from .utils import set_postgres_connection from .utils import sidebar_button # from .utils import th_error @@ -133,11 +139,11 @@ "HBMethod", "HBMethodWindow", "MacroModule", - "MacroModuleTab", - "MacroModuleWindow", "MacroModuleConsolidated", "MacroModuleConsolidatedTab", "MacroModuleConsolidatedWindow", + "MacroModuleTab", + "MacroModuleWindow", "MapDisplay", "MapDisplayTab", "MapDisplayWindow", @@ -163,6 +169,8 @@ "VisualizationBuilder", "VisualizationBuilderWindow", "WindowImplementation", + "_get_connection_callable", + "_get_connection_object", "active_no_duplicates_refnr_list", "app_setup", "apply_edits", @@ -172,12 +180,16 @@ "create_database_engine", "enable_app_logging", "export_from_parqueteditor", + "get_connection", "get_export_log_path", "get_log_path", "ibis_filter_with_dict", "main_layout", "module_validator", "register_control", + "set_connection", + "set_eimerdb_connection", + "set_postgres_connection", "set_variables", "sidebar_button", # "hb_method", diff --git a/src/ssb_dash_framework/control/control_framework_base.py b/src/ssb_dash_framework/control/control_framework_base.py index 6c730405..49731368 100644 --- a/src/ssb_dash_framework/control/control_framework_base.py +++ b/src/ssb_dash_framework/control/control_framework_base.py @@ -4,12 +4,13 @@ from typing import Any from typing import ClassVar -import ibis import pandas as pd from eimerdb import EimerDBInstance from ibis import _ +from psycopg_pool import ConnectionPool -from ..utils.core_query_functions import conn_is_ibis +from ..utils.config_tools.connection import _get_connection_object +from ..utils.config_tools.connection import get_connection from ..utils.core_query_functions import ibis_filter_with_dict logger = logging.getLogger(__name__) @@ -120,9 +121,8 @@ def __init__( self, time_units, applies_to_subset, - conn, ) -> None: - super().__init__(time_units, applies_to_subset, conn) + super().__init__(time_units, applies_to_subset) """ _required_kontroller_columns: ClassVar[list[str]] = [ @@ -142,21 +142,18 @@ def __init__( self, time_units: list[str], applies_to_subset: dict[str, Any], - conn: object, ) -> None: """Initialize the control framework. Args: time_units: Time units that exists in the dataset. applies_to_subset: Subset to execute controls on. - conn: Database connection object. """ self.time_units = time_units self.applies_to_subset = applies_to_subset for key, value in self.applies_to_subset.items(): if not isinstance(value, list): self.applies_to_subset[key] = [value] - self.conn = conn self._required_kontroller_columns = [ *self.time_units, @@ -223,14 +220,14 @@ def register_control(self, control: str) -> None: logger.debug("No new control to register, ending here.") return None logger.debug(f"Rows to register:\n{rows_to_register}") - if isinstance(self.conn, EimerDBInstance): - self.conn.insert("kontroller", rows_to_register) - elif conn_is_ibis(self.conn): - conn = self.conn - conn.insert("kontroller", rows_to_register) + connection_object = _get_connection_object() + if isinstance(connection_object, EimerDBInstance): + connection_object.insert("kontroller", rows_to_register) + elif isinstance(connection_object, ConnectionPool): + connection_object.insert("kontroller", rows_to_register) else: raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + f"Connection type '{type(connection_object)}' is currently not implemented." ) logger.info(f"Done inserting {control}") @@ -245,33 +242,13 @@ def register_all_controls(self) -> None: def get_current_kontroller(self) -> pd.DataFrame | None: """Gets the current contents of the 'kontroller' table.""" logger.info("Getting current contents of table 'kontroller'") - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - try: - kontroller = self.conn.query( - "SELECT * FROM kontroller" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("kontroller", kontroller) - except ( - ValueError - ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents. - if str(e) == "max() arg is an empty sequence": - logger.warning( - "Did not find any contents in 'kontroller', starting from scratch." - ) - return None - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." - ) - kontroller = conn.table("kontroller") - kontroller = kontroller.filter( - ibis_filter_with_dict(self.applies_to_subset) - ).to_pandas() - logger.debug(f"Kontroller data to return:\n{kontroller}") - return kontroller + with get_connection(necessary_tables=["kontroller"]) as conn: + kontroller = conn.table("kontroller") + kontroller = kontroller.filter( + ibis_filter_with_dict(self.applies_to_subset) + ).to_pandas() + logger.debug(f"Kontroller data to return:\n{kontroller}") + return kontroller def execute_controls(self) -> None: """Executes all control methods found in the class.""" @@ -391,47 +368,20 @@ def get_current_kontrollutslag( Returns: pd.DataFrame containing the current kontrollutslag table for all controls or just the specified one or None if table empty. - - Raises: - NotImplementedError: If connection is not EimerDBInstance or Ibis connection. - ValueError: If the query fails for reasons other than an empty 'kontrollutslag' table. """ logger.info("Getting current kontrollutslag.") - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - try: - kontrollutslag = self.conn.query( - "SELECT * FROM kontrollutslag" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("kontrollutslag", kontrollutslag) - except ( - ValueError - ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents. - if str(e) == "max() arg is an empty sequence": - logger.warning( - "Did not find any contents in 'kontrollutslag', starting from scratch." - ) - return None - else: - raise e - - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + with get_connection(necessary_tables=["kontrollutslag"]) as conn: + kontrollutslag = conn.table("kontrollutslag") + kontrollutslag = kontrollutslag.filter( + ibis_filter_with_dict(self.applies_to_subset) ) - kontrollutslag = conn.table("kontrollutslag") - kontrollutslag = kontrollutslag.filter( - ibis_filter_with_dict(self.applies_to_subset) - ) - if specific_control: - kontrollutslag = kontrollutslag.filter(_.kontrollid == specific_control) - kontrollutslag = kontrollutslag.to_pandas() - logger.debug( - f"Existing kontrollutslag\nAmount:{kontrollutslag['utslag'].value_counts()}\nData:\n{kontrollutslag}" - ) - return kontrollutslag + if specific_control: + kontrollutslag = kontrollutslag.filter(_.kontrollid == specific_control) + kontrollutslag = kontrollutslag.to_pandas() + logger.debug( + f"Existing kontrollutslag\nAmount:{kontrollutslag['utslag'].value_counts()}\nData:\n{kontrollutslag}" + ) + return kontrollutslag def insert_new_records(self, control_results: pd.DataFrame) -> None: """Inserts new records that are not found in the current contents of the 'kontrollutslag' table.""" @@ -471,14 +421,14 @@ def insert_new_records(self, control_results: pd.DataFrame) -> None: return None # Now to insert new rows into the table. logger.debug(f"Inserting {control_results.shape[0]} new rows.") - if isinstance(self.conn, EimerDBInstance): - self.conn.insert("kontrollutslag", control_results) - elif conn_is_ibis(self.conn): - conn = self.conn - conn.insert("kontrollutslag", control_results) + connection_object = _get_connection_object() + if isinstance(connection_object, EimerDBInstance): + connection_object.insert("kontrollutslag", control_results) + elif isinstance(connection_object, ConnectionPool): + connection_object.insert("kontrollutslag", control_results) else: raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + f"Connection type '{type(connection_object)}' is currently not implemented." ) logger.debug("Finished inserting new rows.") @@ -546,15 +496,18 @@ def update_existing_records(self, control_results: pd.DataFrame) -> None: logger.info(f"Updating {changed.shape[0]} rows.") logger.debug(f"Rows to update:\n{changed}") update_query = self.generate_update_query(changed) - if isinstance(self.conn, EimerDBInstance): - self.conn.query(update_query) - elif conn_is_ibis(self.conn): - conn = self.conn - conn.raw_sql(update_query) # type: ignore[attr-defined] + + connection_object = _get_connection_object() + if isinstance(connection_object, EimerDBInstance): + connection_object.query(update_query) + elif isinstance(connection_object, ConnectionPool): + with get_connection() as conn: + conn.raw_sql(update_query) else: raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + f"Connection type '{type(connection_object)}' is currently not implemented." ) + logger.debug("Finished updating kontrollutslag.") def generate_update_query(self, df_updates: pd.DataFrame) -> str: diff --git a/src/ssb_dash_framework/modules/agg_dist_plotter.py b/src/ssb_dash_framework/modules/agg_dist_plotter.py index 7e034701..bed1caac 100644 --- a/src/ssb_dash_framework/modules/agg_dist_plotter.py +++ b/src/ssb_dash_framework/modules/agg_dist_plotter.py @@ -16,13 +16,12 @@ from dash import dcc from dash import html from dash.exceptions import PreventUpdate -from eimerdb import EimerDBInstance from ..setup.variableselector import VariableSelector from ..utils import TabImplementation from ..utils import WindowImplementation from ..utils import active_no_duplicates_refnr_list -from ..utils import conn_is_ibis +from ..utils import get_connection from ..utils.eimerdb_helpers import create_partition_select from ..utils.module_validation import module_validator @@ -62,7 +61,9 @@ class AggDistPlotter(ABC): ] ) - def __init__(self, time_units: list[str], conn: object) -> None: + def __init__( + self, time_units: list[str], main_table_name="skjemadata_hoved" + ) -> None: """Initializes the AggDistPlotter. Args: @@ -90,8 +91,7 @@ def __init__(self, time_units: list[str], conn: object) -> None: for x in time_units ] print("TIME UNITS ", self.time_units) - - self.conn = conn + self.main_table_name = main_table_name self.module_layout = self._create_layout() self.module_callbacks() @@ -328,82 +328,60 @@ def agg_table( ) _t_0 = str(time_vars[0]) _t_1 = str(time_vars[1]) - if isinstance(self.conn, EimerDBInstance): - _t_0 = int(_t_0) - _t_1 = int(_t_1) - conn = ibis.polars.connect() - - skjemamottak = self.conn.query( - "SELECT * FROM skjemamottak", - partition_select=updated_partition_select, - ) - skjemadata = self.conn.query( - "SELECT * FROM skjemadata_hoved", - partition_select=updated_partition_select, - ) - datatyper = self.conn.query( - "SELECT * FROM datatyper", partition_select=updated_partition_select - ) - - conn.create_table("skjemamottak", skjemamottak) - conn.create_table("skjemadata_hoved", skjemadata) - conn.create_table("datatyper", datatyper) - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." - ) - - skjemadata_tbl = conn.table("skjemadata_hoved") - datatyper_tbl = conn.table("datatyper") - relevant_refnr = active_no_duplicates_refnr_list(conn, skjema) - - skjemadata_tbl = ( - skjemadata_tbl.filter(skjemadata_tbl.refnr.isin(relevant_refnr)) - .join( - datatyper_tbl.select("variabel", "datatype"), - ["variabel"], - how="inner", - ) - .filter(datatyper_tbl.datatype.isin(["number", "int", "float"])) - .cast({"verdi": "float", rullerende_var: "str"}) - .cast({"verdi": "int"}) - .mutate(verdi=lambda t: t["verdi"].round(0)) - .pivot_wider( - id_cols=["variabel"], - names_from=rullerende_var, # TODO: Tidsenhet - values_from="verdi", - values_agg="sum", + with get_connection( # necessary_tables and partition_select are used for eimerdb connection. + necessary_tables=["skjemamottak", "datatyper", self.main_table_name], + partition_select=updated_partition_select, + ) as conn: + skjemadata_tbl = conn.table(self.main_table_name) + datatyper_tbl = conn.table("datatyper") + + relevant_refnr = active_no_duplicates_refnr_list(conn, skjema) + + skjemadata_tbl = ( + skjemadata_tbl.filter(skjemadata_tbl.refnr.isin(relevant_refnr)) + .join( + datatyper_tbl.select("variabel", "datatype"), + ["variabel"], + how="inner", + ) + .filter(datatyper_tbl.datatype.isin(["number", "int", "float"])) + .cast({"verdi": "float", rullerende_var: "str"}) + .cast({"verdi": "int"}) + .mutate(verdi=lambda t: t["verdi"].round(0)) + .pivot_wider( + id_cols=["variabel"], + names_from=rullerende_var, # TODO: Tidsenhet + values_from="verdi", + values_agg="sum", + ) ) - ) - if _t_1 in skjemadata_tbl.columns: - logger.debug("Calculating diff from last year.") - skjemadata_tbl.mutate( - diff=lambda t: t[_t_0] - t[_t_1], - pdiff=lambda t: ( - (t[_t_0].fill_null(0) - t[_t_1].fill_null(0)) - / t[_t_1].fill_null(1) - * 100 - ).round(2), - ) - else: - logger.debug( - f"Didn't find previous period value, no diff calculated. Columns in dataset: {skjemadata_tbl.columns}" - ) - - pandas_table = skjemadata_tbl.to_pandas() - columns = [ - { - "headerName": col, - "field": col, - } - for col in pandas_table.columns - ] - columns[0]["checkboxSelection"] = True - columns[0]["headerCheckboxSelection"] = True - return pandas_table.to_dict("records"), columns + if _t_1 in skjemadata_tbl.columns: + logger.debug("Calculating diff from last year.") + skjemadata_tbl.mutate( + diff=lambda t: t[_t_0] - t[_t_1], + pdiff=lambda t: ( + (t[_t_0].fill_null(0) - t[_t_1].fill_null(0)) + / t[_t_1].fill_null(1) + * 100 + ).round(2), + ) + else: + logger.debug( + f"Didn't find previous period value, no diff calculated. Columns in dataset: {skjemadata_tbl.columns}" + ) + + pandas_table = skjemadata_tbl.to_pandas() + columns = [ + { + "headerName": col, + "field": col, + } + for col in pandas_table.columns + ] + columns[0]["checkboxSelection"] = True + columns[0]["headerCheckboxSelection"] = True + return pandas_table.to_dict("records"), columns @callback( # type: ignore[misc] Output("aggdistplotter-graph", "figure"), @@ -452,111 +430,98 @@ def agg_graph1( **partition_args, ) - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - skjemamottak = self.conn.query( - "SELECT * FROM skjemamottak", partition_select=partition_select - ) - skjemadata = self.conn.query( - "SELECT * FROM skjemadata_hoved", partition_select=partition_select + with get_connection( + necessary_tables=["skjemamottak", self.main_table_name], + partition_select=partition_select, + ) as conn: + skjemamottak_tbl = conn.table("skjemamottak") + skjemadata_tbl = conn.table(self.main_table_name) + + skjemamottak_tbl = ( # Get relevant refnr values from skjemamottak + skjemamottak_tbl.filter(skjemamottak_tbl.aktiv) + .order_by(ibis.desc(skjemamottak_tbl.dato_mottatt)) + .distinct(on=[*self.time_units, "ident"], keep="first") ) - conn.create_table("skjemamottak", skjemamottak) - conn.create_table("skjemadata_hoved", skjemadata) - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + relevant_refnr = active_no_duplicates_refnr_list(conn, skjema) + + skjemadata_tbl = ( + skjemadata_tbl.filter( + [ + skjemadata_tbl.refnr.isin(relevant_refnr), + skjemadata_tbl.variabel == variabel, + skjemadata_tbl.verdi.notnull(), + ] + ) + .cast({"verdi": "float"}) + .cast({"verdi": "int"}) ) - skjemamottak_tbl = conn.table("skjemamottak") - skjemadata_tbl = conn.table("skjemadata_hoved") - - skjemamottak_tbl = ( # Get relevant refnr values from skjemamottak - skjemamottak_tbl.filter(skjemamottak_tbl.aktiv) - .order_by(ibis.desc(skjemamottak_tbl.dato_mottatt)) - .distinct(on=[*self.time_units, "ident"], keep="first") - ) - relevant_refnr = active_no_duplicates_refnr_list(conn, skjema) - - skjemadata_tbl = ( - skjemadata_tbl.filter( - [ - skjemadata_tbl.refnr.isin(relevant_refnr), - skjemadata_tbl.variabel == variabel, - skjemadata_tbl.verdi.notnull(), - ] - ) - .cast({"verdi": "float"}) - .cast({"verdi": "int"}) - ) - - df = skjemadata_tbl.to_pandas() - - top5_df = df.nlargest(5, "verdi") - - if graph_type == "box": - fig = px.box( - df, - x="variabel", - y="verdi", - hover_data=["ident", "verdi"], - points="all", - title=f"📦 Boksplott for {variabel}, {partition_select!s}.", - template="plotly_dark", - ) - elif graph_type == "fiolin": - fig = px.violin( - df, - x="variabel", - y="verdi", - hover_data=["ident", "verdi"], - box=True, - points="all", - title=f"🎻 Fiolinplott for {variabel}, {partition_select!s}.", - template="plotly_dark", - ) - elif graph_type == "bidrag": - agg_df = df.groupby("ident", as_index=False)["verdi"].sum() - agg_df["verdi"] = (agg_df["verdi"] / agg_df["verdi"].sum() * 100).round( - 2 - ) - agg_df = agg_df.sort_values("verdi", ascending=False).head(10) - - fig = px.bar( - agg_df, - x="verdi", - y="ident", - orientation="h", - title=f"🥇 Bidragsanalyse - % av total verdi ({variabel})", - template="plotly_dark", - labels={"verdi": "%"}, - custom_data=["ident"], - ) - - fig.update_layout(yaxis={"categoryorder": "total ascending"}) - - else: - fig = go.Figure() - - if graph_type in ["box", "fiolin"]: - fig.add_scatter( - x=[variabel] * len(top5_df), - y=top5_df["verdi"], - mode="markers", - marker=dict( - size=13, - color="#00CC96", - symbol="diamond", - line=dict(width=1, color="white"), - ), - name="De fem største", - hovertext=top5_df["ident"], - hoverinfo="text+y", - customdata=top5_df[["ident"]].values, - ) - return fig + df = skjemadata_tbl.to_pandas() + + top5_df = df.nlargest(5, "verdi") + + if graph_type == "box": + fig = px.box( + df, + x="variabel", + y="verdi", + hover_data=["ident", "verdi"], + points="all", + title=f"📦 Boksplott for {variabel}, {partition_select!s}.", + template="plotly_dark", + ) + elif graph_type == "fiolin": + fig = px.violin( + df, + x="variabel", + y="verdi", + hover_data=["ident", "verdi"], + box=True, + points="all", + title=f"🎻 Fiolinplott for {variabel}, {partition_select!s}.", + template="plotly_dark", + ) + elif graph_type == "bidrag": + agg_df = df.groupby("ident", as_index=False)["verdi"].sum() + agg_df["verdi"] = ( + agg_df["verdi"] / agg_df["verdi"].sum() * 100 + ).round(2) + agg_df = agg_df.sort_values("verdi", ascending=False).head(10) + + fig = px.bar( + agg_df, + x="verdi", + y="ident", + orientation="h", + title=f"🥇 Bidragsanalyse - % av total verdi ({variabel})", + template="plotly_dark", + labels={"verdi": "%"}, + custom_data=["ident"], + ) + + fig.update_layout(yaxis={"categoryorder": "total ascending"}) + + else: + fig = go.Figure() + + if graph_type in ["box", "fiolin"]: + fig.add_scatter( + x=[variabel] * len(top5_df), + y=top5_df["verdi"], + mode="markers", + marker=dict( + size=13, + color="#00CC96", + symbol="diamond", + line=dict(width=1, color="white"), + ), + name="De fem største", + hovertext=top5_df["ident"], + hoverinfo="text+y", + customdata=top5_df[["ident"]].values, + ) + return fig @callback( # type: ignore[misc] Output("var-ident", "value", allow_duplicate=True), @@ -575,16 +540,24 @@ def output_to_variabelvelger(clickdata: dict[str, list[dict[str, Any]]]) -> str: class AggDistPlotterTab(TabImplementation, AggDistPlotter): """AggDistPlotterTab is an implementation of the AggDistPlotter module as a tab in a Dash application.""" - def __init__(self, time_units: list[str], conn: object) -> None: + def __init__( + self, time_units: list[str], main_table_name="skjemadata_hoved" + ) -> None: """Initializes the AggDistPlotterTab class.""" - AggDistPlotter.__init__(self, time_units=time_units, conn=conn) + AggDistPlotter.__init__( + self, time_units=time_units, main_table_name=main_table_name + ) TabImplementation.__init__(self) class AggDistPlotterWindow(WindowImplementation, AggDistPlotter): """AggDistPlotterWindow is an implementation of the AggDistPlotter module as a tab in a Dash application.""" - def __init__(self, time_units: list[str], conn: object) -> None: + def __init__( + self, time_units: list[str], main_table_name="skjemadata_hoved" + ) -> None: """Initializes the AggDistPlotterWindow class.""" - AggDistPlotter.__init__(self, time_units=time_units, conn=conn) + AggDistPlotter.__init__( + self, time_units=time_units, main_table_name=main_table_name + ) WindowImplementation.__init__(self) diff --git a/src/ssb_dash_framework/modules/altinn_control_view.py b/src/ssb_dash_framework/modules/altinn_control_view.py index a7f4a97d..fc330761 100644 --- a/src/ssb_dash_framework/modules/altinn_control_view.py +++ b/src/ssb_dash_framework/modules/altinn_control_view.py @@ -20,7 +20,8 @@ from ..utils import TabImplementation from ..utils import WindowImplementation from ..utils.alert_handler import create_alert -from ..utils.core_query_functions import conn_is_ibis +from ..utils.config_tools.connection import _get_connection_object +from ..utils.config_tools.connection import get_connection from ..utils.module_validation import module_validator logger = logging.getLogger(__name__) @@ -44,7 +45,6 @@ def __init__( self, time_units: list[str], control_dict: dict[str, Any], - conn: object, outputs: list[str] | None = None, ) -> None: # TODO add proper annotation for control_dict value """Initializes the ControlView with time units, control dictionary, and database connection. @@ -52,21 +52,13 @@ def __init__( Args: time_units: A list of the time units used. control_dict: A dictionary with one control class per skjema. - conn: The eimerdb connection. outputs: Variable selector fields to output to. Defaults to ['ident'] - - Raises: - TypeError: if conn type is not 'EimerDBInstance' or ibis connection. """ logger.warning( f"{self.__class__.__name__} is under development and may change in future releases." ) if outputs is None: outputs = ["ident"] - if not conn_is_ibis(conn) and not isinstance(conn, EimerDBInstance): - raise TypeError( - f"Argument 'conn' must be an 'EimerDBInstance' or Ibis backend. Received: {type(conn)}" - ) self.module_number = ControlView._id_number self.module_name = self.__class__.__name__ ControlView._id_number += 1 @@ -75,7 +67,6 @@ def __init__( self.label = "Kontroll" self.control_dict = control_dict - self.conn = conn self.outputs = outputs self._is_valid() self.module_layout = self.create_layout() @@ -226,14 +217,14 @@ def get_kontroller_overview( f"rerun: {rerun}\n" f"args: {args}" ) - if isinstance(self.conn, EimerDBInstance): + + if isinstance(_get_connection_object(), EimerDBInstance): args = [int(arg) for arg in args] logger.debug(dict(zip(self.time_units, args, strict=False))) control_class_instance = self.control_dict[skjema]( time_units=self.time_units, applies_to_subset=dict(zip(self.time_units, args, strict=False)) | {"skjema": [skjema]}, - conn=self.conn, ) if ( ctx.triggered_id == f"{self.module_number}-kontroll-run-button" @@ -261,122 +252,84 @@ def get_kontroller_overview( raise e else: logger.info("Refreshing view without re-running controls.") - - if isinstance(self.conn, EimerDBInstance): - try: - conn = ibis.polars.connect() - skjemamottak = self.conn.query( - "SELECT * FROM skjemamottak" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("skjemamottak", skjemamottak) - kontroller = self.conn.query( - "SELECT * FROM kontroller" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("kontroller", kontroller) - kontrollutslag = self.conn.query( - "SELECT * FROM kontrollutslag" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("kontrollutslag", kontrollutslag) - except ( - ValueError - ) as e: # TODO permanently fix this. Error caused by running .query on eimerdb table with no contents. - if str(e) == "max() arg is an empty sequence": - logger.warning( - "Did not find any contents in control tables, returning None, None and alert." - ) - alert_store = [ - create_alert( - "Finner ingen kontroller i dataene, prøv å kjøre kontroller.", - "warning", - ephemeral=True, - ), - *alert_store, - ] - return None, None, alert_store - else: - raise e - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." + with get_connection( + necessary_tables=["skjemamottak", "kontroller", "kontrollutslag"] + ) as conn: + + skjemamottak = conn.table("skjemamottak") + kontroller = conn.table("kontroller") + kontrollutslag = conn.table("kontrollutslag") + + utslag = ( + kontrollutslag.filter(kontrollutslag.utslag == True) + .group_by(kontrollutslag.kontrollid) + .aggregate(ant_utslag=ibis._.count()) ) - # TODO make sure conn is defined for mypy. - skjemamottak = conn.table("skjemamottak") - kontroller = conn.table("kontroller") - kontrollutslag = conn.table("kontrollutslag") - - utslag = ( - kontrollutslag.filter(kontrollutslag.utslag == True) - .group_by(kontrollutslag.kontrollid) - .aggregate(ant_utslag=ibis._.count()) - ) - - subq = ( - skjemamottak.filter(skjemamottak.aktiv == True) - .filter(skjemamottak.editert == False) - .select("ident", "refnr") - ) + subq = ( + skjemamottak.filter(skjemamottak.aktiv == True) + .filter(skjemamottak.editert == False) + .select("ident", "refnr") + ) - uediterte = ( - kontrollutslag.join( - subq, - (kontrollutslag.ident == subq.ident) - & (kontrollutslag.refnr == subq.refnr), + uediterte = ( + kontrollutslag.join( + subq, + (kontrollutslag.ident == subq.ident) + & (kontrollutslag.refnr == subq.refnr), + ) + .filter(kontrollutslag.utslag == True) + .group_by(kontrollutslag.kontrollid) + .aggregate(uediterte=ibis._.count()) + .select("kontrollid", "uediterte") ) - .filter(kontrollutslag.utslag == True) - .group_by(kontrollutslag.kontrollid) - .aggregate(uediterte=ibis._.count()) - .select("kontrollid", "uediterte") - ) - result = ( - kontroller.join(utslag, kontroller.kontrollid == utslag.kontrollid) - .join(uediterte, kontroller.kontrollid == uediterte.kontrollid) - .select( - kontroller.skjema, - kontroller.kontrollid, - kontroller.type, - kontroller.beskrivelse, - kontroller.sorting_var, - kontroller.sorting_order, - utslag.ant_utslag, - uediterte.uediterte, + result = ( + kontroller.join(utslag, kontroller.kontrollid == utslag.kontrollid) + .join(uediterte, kontroller.kontrollid == uediterte.kontrollid) + .select( + kontroller.skjema, + kontroller.kontrollid, + kontroller.type, + kontroller.beskrivelse, + kontroller.sorting_var, + kontroller.sorting_order, + utslag.ant_utslag, + uediterte.uediterte, + ) ) - ) - result = result.to_pandas().sort_values(by="kontrollid", ascending=True) - columns = [ - { - "headerName": col, - "field": col, - "hide": col == "sorting_order", - "flex": 2 if col == "beskrivelse" else 1, - "tooltipField": col if col == "beskrivelse" else None, - } - for col in result.columns - ] - columns[0]["checkboxSelection"] = True - columns[0]["headerCheckboxSelection"] = True - if ctx.triggered_id == f"{self.module_number}-kontroll-run-button": - alert_store = [ - create_alert( - f"Kontrollkjøring ferdig for kontroller i {control_class_instance.__class__.__name__}", - "info", - ephemeral=True, - ), - *alert_store, + result = result.to_pandas().sort_values(by="kontrollid", ascending=True) + columns = [ + { + "headerName": col, + "field": col, + "hide": col == "sorting_order", + "flex": 2 if col == "beskrivelse" else 1, + "tooltipField": col if col == "beskrivelse" else None, + } + for col in result.columns ] - else: - alert_store = [ - create_alert( - "Kontrollvisning oppdatert.", - "info", - ephemeral=True, - ), - *alert_store, - ] - return result.to_dict("records"), columns, alert_store + columns[0]["checkboxSelection"] = True + columns[0]["headerCheckboxSelection"] = True + if ctx.triggered_id == f"{self.module_number}-kontroll-run-button": + alert_store = [ + create_alert( + f"Kontrollkjøring ferdig for kontroller i {control_class_instance.__class__.__name__}", + "info", + ephemeral=True, + ), + *alert_store, + ] + else: + alert_store = [ + create_alert( + "Kontrollvisning oppdatert.", + "info", + ephemeral=True, + ), + *alert_store, + ] + return result.to_dict("records"), columns, alert_store @callback( # type: ignore[misc] Output(f"{self.module_number}-kontrollutslag", "rowData"), @@ -390,63 +343,50 @@ def get_kontrollutslag(current_row: list[dict[Any, Any]], *args: Any): if current_row is None or len(current_row) == 0: logger.debug("No current_row, raising PreventUpdate.") raise PreventUpdate - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - skjemamottak = self.conn.query( - "SELECT * FROM skjemamottak" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("skjemamottak", skjemamottak) - kontrollutslag = self.conn.query( - "SELECT * FROM kontrollutslag" - ) # maybe add something like this?partition_select=self.applies_to_subset - conn.create_table("kontrollutslag", kontrollutslag) - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise NotImplementedError( - f"Connection type '{type(self.conn)}' is currently not implemented." - ) - kontrollid = current_row[0]["kontrollid"] - sorting_var = current_row[0]["sorting_var"] - skjema = current_row[0]["skjema"] - sorting_order = current_row[0]["sorting_order"] - - logger.debug( - f"Variables from current_row:\nkontrollid: {kontrollid}\nsorting_var: {sorting_var}\nskjema: {skjema}\nsorting_order: {sorting_order}" - ) - if sorting_order is None: - sorting_order = "DESC" - skjemamottak = conn.table("skjemamottak") - kontrollutslag = conn.table("kontrollutslag") - # Subquery: filter active rows in skjemamottak - s = skjemamottak.filter(skjemamottak.aktiv == True).select( - skjemamottak.refnr, - skjemamottak.editert, - skjemamottak.ident, - ) - - # Main query - result = ( - kontrollutslag.join( - s, - (kontrollutslag.refnr == s.refnr) - & (kontrollutslag.ident == s.ident), + with get_connection( + necessary_tables=["skjemamottak", "kontrollutslag"] + ) as conn: + kontrollid = current_row[0]["kontrollid"] + sorting_var = current_row[0]["sorting_var"] + skjema = current_row[0]["skjema"] + sorting_order = current_row[0]["sorting_order"] + + logger.debug( + f"Variables from current_row:\nkontrollid: {kontrollid}\nsorting_var: {sorting_var}\nskjema: {skjema}\nsorting_order: {sorting_order}" ) - .filter( - (kontrollutslag.kontrollid == kontrollid) - & (kontrollutslag.utslag == True) + if sorting_order is None: + sorting_order = "DESC" + skjemamottak = conn.table("skjemamottak") + kontrollutslag = conn.table("kontrollutslag") + # Subquery: filter active rows in skjemamottak + s = skjemamottak.filter(skjemamottak.aktiv == True).select( + skjemamottak.refnr, + skjemamottak.editert, + skjemamottak.ident, ) - .select( - kontrollutslag.ident, - kontrollutslag.refnr, - kontrollutslag.kontrollid, - kontrollutslag.utslag, - s.editert, - kontrollutslag.verdi, + + # Main query + result = ( + kontrollutslag.join( + s, + (kontrollutslag.refnr == s.refnr) + & (kontrollutslag.ident == s.ident), + ) + .filter( + (kontrollutslag.kontrollid == kontrollid) + & (kontrollutslag.utslag == True) + ) + .select( + kontrollutslag.ident, + kontrollutslag.refnr, + kontrollutslag.kontrollid, + kontrollutslag.utslag, + s.editert, + kontrollutslag.verdi, + ) + .order_by(s.editert, kontrollutslag.verdi) ) - .order_by(s.editert, kontrollutslag.verdi) - ) - result = result.to_pandas() + result = result.to_pandas() columns = [{"headerName": col, "field": col} for col in result.columns] columns[0]["checkboxSelection"] = True columns[0]["headerCheckboxSelection"] = True @@ -485,14 +425,15 @@ class ControlViewTab(TabImplementation, ControlView): """ControlView implemented as a tab.""" def __init__( - self, time_units: list[str], control_dict: dict[str, Any], conn: object + self, + time_units: list[str], + control_dict: dict[str, Any], ) -> None: """Initializes the ControlViewTab module.""" ControlView.__init__( self, time_units=time_units, control_dict=control_dict, - conn=conn, ) TabImplementation.__init__(self) @@ -501,14 +442,15 @@ class ControlViewWindow(WindowImplementation, ControlView): """ControlView implemented as a window.""" def __init__( - self, time_units: list[str], control_dict: dict[str, Any], conn: object + self, + time_units: list[str], + control_dict: dict[str, Any], ) -> None: """Initializes the ControlViewWindow module.""" ControlView.__init__( self, time_units=time_units, control_dict=control_dict, - conn=conn, ) WindowImplementation.__init__(self) @@ -517,9 +459,7 @@ def __init__( class AltinnControlViewTab(TabImplementation, ControlView): """ControlView implemented as a tab.""" - def __init__( - self, time_units: list[str], control_dict: dict[str, Any], conn: object - ) -> None: + def __init__(self, time_units: list[str], control_dict: dict[str, Any]) -> None: """Initializes the ControlViewTab module.""" warnings.warn( "AltinnControlViewTab is deprecated and will be removed in a future version. " @@ -531,7 +471,6 @@ def __init__( self, time_units=time_units, control_dict=control_dict, - conn=conn, ) TabImplementation.__init__(self) @@ -540,7 +479,9 @@ class AltinnControlViewWindow(WindowImplementation, ControlView): """ControlView implemented as a window.""" def __init__( - self, time_units: list[str], control_dict: dict[str, Any], conn: object + self, + time_units: list[str], + control_dict: dict[str, Any], ) -> None: """Initializes the ControlViewWindow module.""" warnings.warn( @@ -553,6 +494,5 @@ def __init__( self, time_units=time_units, control_dict=control_dict, - conn=conn, ) WindowImplementation.__init__(self) diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_comment.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_comment.py index 0cfb4fae..dab829c2 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_comment.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_comment.py @@ -1,10 +1,8 @@ import logging -import time from typing import Any import dash_ag_grid as dag import dash_bootstrap_components as dbc -import ibis from dash import callback from dash import dcc from dash import html @@ -18,6 +16,7 @@ from ssb_dash_framework.utils import conn_is_ibis from ...utils import create_alert +from ...utils import get_connection logger = logging.getLogger(__name__) @@ -27,7 +26,6 @@ class AltinnEditorComment: def __init__( self, - conn: object, ) -> None: """Initializes the Altinn Editor Comment module. @@ -37,11 +35,6 @@ def __init__( Raises: TypeError: If the connection object is not 'EimerDBInstance' or ibis connection. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn self.module_layout = self._create_layout() self.module_callbacks() @@ -155,25 +148,18 @@ def kommentar_table( if n_clicks is None: logger.debug("Raised PreventUpdate") raise PreventUpdate - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM skjemamottak") - conn.create_table("skjemamottak", data) - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise TypeError("Connection object is invalid type.") - time.sleep(2) - t = conn.table("skjemamottak") - df = t.filter(_.ident == ident).filter(_.skjema == skjema).to_pandas() - columns = [ - { - "headerName": col, - "field": col, - } - for col in df.columns - ] - return df.to_dict("records"), columns + + with get_connection(necessary_tables=["skjemamottak"]) as conn: + t = conn.table("skjemamottak") + df = t.filter(_.ident == ident).filter(_.skjema == skjema).to_pandas() + columns = [ + { + "headerName": col, + "field": col, + } + for col in df.columns + ] + return df.to_dict("records"), columns @callback( # type: ignore[misc] Output("skjemadata-kommentarmodal-aar-kommentar", "value"), diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_contact.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_contact.py index c4f036a8..0a5fad40 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_contact.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_contact.py @@ -2,19 +2,15 @@ from typing import Any import dash_bootstrap_components as dbc -import ibis from dash import callback from dash import html from dash.dependencies import Input from dash.dependencies import Output from dash.dependencies import State from dash.exceptions import PreventUpdate -from eimerdb import EimerDBInstance from ibis import _ -from ssb_dash_framework.utils import conn_is_ibis -from ssb_dash_framework.utils import create_filter_dict -from ssb_dash_framework.utils import ibis_filter_with_dict +from ssb_dash_framework.utils import get_connection from ...setup.variableselector import VariableSelector @@ -27,24 +23,17 @@ class AltinnEditorContact: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, ) -> None: """Initializes the Altinn Editor Contact module. Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. variable_selector_instance: An instance of VariableSelector for variable selection. Raises: TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -198,39 +187,30 @@ def kontaktinfocanvas( f"args: {args}" ) - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM kontaktinfo") - conn.create_table("kontaktinfo", data) - filter_dict = create_filter_dict( - self.time_units, [int(x) for x in args] - ) - elif conn_is_ibis(self.conn): - conn = self.conn - filter_dict = create_filter_dict(self.time_units, args) - else: - raise TypeError("Connection object is invalid type.") - t = conn.table("kontaktinfo") - df_skjemainfo = ( - t.filter(_.refnr == refnr) - .filter(ibis_filter_with_dict(filter_dict)) - .select( - [ - "kontaktperson", - "epost", - "telefon", - "kommentar_kontaktinfo", - "kommentar_krevende", - ] + with get_connection( # necessary_tables and partition_select are used for eimerdb connection. + necessary_tables=["kontaktinfo"], + ) as conn: + + t = conn.table("kontaktinfo") + df_skjemainfo = ( + t.filter(_.refnr == refnr) + .select( + [ + "kontaktperson", + "epost", + "telefon", + "kommentar_kontaktinfo", + "kommentar_krevende", + ] + ) + .to_pandas() ) - .to_pandas() - ) - if df_skjemainfo.empty: - logger.info("Kontaktinfo table for ") - kontaktperson = df_skjemainfo["kontaktperson"][0] - epost = df_skjemainfo["epost"][0] - telefon = df_skjemainfo["telefon"][0] - kommentar1 = df_skjemainfo["kommentar_kontaktinfo"][0] - kommentar2 = df_skjemainfo["kommentar_krevende"][0] - return kontaktperson, epost, telefon, kommentar1, kommentar2 + if df_skjemainfo.empty: + logger.info("Kontaktinfo table for ") + kontaktperson = df_skjemainfo["kontaktperson"][0] + epost = df_skjemainfo["epost"][0] + telefon = df_skjemainfo["telefon"][0] + kommentar1 = df_skjemainfo["kommentar_kontaktinfo"][0] + kommentar2 = df_skjemainfo["kommentar_krevende"][0] + return kontaktperson, epost, telefon, kommentar1, kommentar2 diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_control.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_control.py index f7ed16ae..bf10ca69 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_control.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_control.py @@ -1,23 +1,18 @@ import logging -import time from typing import Any import dash_ag_grid as dag import dash_bootstrap_components as dbc -import ibis from dash import callback from dash import html from dash.dependencies import Input from dash.dependencies import Output from dash.dependencies import State from dash.exceptions import PreventUpdate -from eimerdb import EimerDBInstance from ibis import _ -from ssb_dash_framework.utils import conn_is_ibis -from ssb_dash_framework.utils import ibis_filter_with_dict - from ...setup.variableselector import VariableSelector +from ...utils.config_tools.connection import get_connection logger = logging.getLogger(__name__) @@ -28,24 +23,17 @@ class AltinnEditorControl: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, ) -> None: """Initializes the Altinn Editor Control module. Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. variable_selector_instance: An instance of VariableSelector for variable selection. Raises: TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -152,58 +140,33 @@ def kontrollutslagstabell( or any(arg is None for arg in args) ): return None, None, None, "Se kontrollutslag" - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM kontroller") - conn.create_table("kontroller", data) - kontrollutslag = self.conn.query("SELECT * FROM kontrollutslag") - conn.create_table("kontrollutslag", kontrollutslag) - elif conn_is_ibis(self.conn): - conn = self.conn - else: - raise TypeError("Connection object is invalid type.") - try: - filter_dict = {"aar": "2024"} - k = conn.table("kontroller") - u = conn.table("kontrollutslag") - refnr = selected_row[0]["refnr"] - time.sleep(1.5) - df = ( - u.filter(_.refnr == refnr) - .filter(_.utslag == True) - .filter(ibis_filter_with_dict(filter_dict)) - .join(k, "kontrollid", how="left") - .select("kontrollid", "beskrivelse", "utslag") - .to_pandas() - ) - - # partition_args = dict(zip(self.time_units, args, strict=False)) - # df = self.conn.query( - # f"""SELECT t1.kontrollid, subquery.beskrivelse, t1.utslag - # FROM kontrollutslag AS t1 - # JOIN ( - # SELECT t2.kontrollid, t2.beskrivelse - # FROM kontroller AS t2 - # ) AS subquery ON t1.kontrollid = subquery.kontrollid - # WHERE refnr = '{refnr}' - # AND utslag = True""", - # partition_select=create_partition_select( - # desired_partitions=self.time_units, - # skjema=skjema, - # **partition_args, - # ), - # ) - columns = [{"headerName": col, "field": col} for col in df.columns] - antall_utslag = len(df) - - if antall_utslag > 0: - style = {"color": "#dc3545", "background-color": "#343a40"} - button_text = f"Se kontrollutslag ({antall_utslag})" - else: - style = None - button_text = "Se kontrollutslag" - - return df.to_dict("records"), columns, style, button_text - except Exception as e: - logger.error(f"Error in kontrollutslagstabell: {e}", exc_info=True) - return None, None, None, "Se kontrollutslag" + + with get_connection( + necessary_tables=["kontroller", "kontrollutslag"] + ) as conn: + try: + k = conn.table("kontroller") + u = conn.table("kontrollutslag") + refnr = selected_row[0]["refnr"] + df = ( + u.filter(_.refnr == refnr) + .filter(_.utslag == True) + .join(k, "kontrollid", how="left") + .select("kontrollid", "beskrivelse", "utslag") + .to_pandas() + ) + + columns = [{"headerName": col, "field": col} for col in df.columns] + antall_utslag = len(df) + + if antall_utslag > 0: + style = {"color": "#dc3545", "background-color": "#343a40"} + button_text = f"Se kontrollutslag ({antall_utslag})" + else: + style = None + button_text = "Se kontrollutslag" + + return df.to_dict("records"), columns, style, button_text + except Exception as e: + logger.error(f"Error in kontrollutslagstabell: {e}", exc_info=True) + return None, None, None, "Se kontrollutslag" diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_history.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_history.py index c2fc26a2..84e4019f 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_history.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_history.py @@ -4,6 +4,7 @@ import dash_ag_grid as dag import dash_bootstrap_components as dbc import pandas as pd +import tzlocal from dash import callback from dash import html from dash.dependencies import Input @@ -12,11 +13,11 @@ from dash.exceptions import PreventUpdate from eimerdb import EimerDBInstance from ibis import _ -import tzlocal - -from ssb_dash_framework.utils import conn_is_ibis +from psycopg_pool import ConnectionPool from ...setup.variableselector import VariableSelector +from ...utils.config_tools.connection import _get_connection_object +from ...utils.config_tools.connection import get_connection from ...utils.eimerdb_helpers import create_partition_select logger = logging.getLogger(__name__) @@ -30,23 +31,17 @@ class AltinnEditorHistory: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, ) -> None: """Initializes the Altinn Editor History module. Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query_changes' method. variable_selector_instance: An instance of VariableSelector for variable selection. Raises: TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - # assert hasattr( - # conn, "query_changes" - # ), "The database object must have a 'query_changes' method." - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -152,11 +147,15 @@ def historikktabell( f"skjema: {skjema}\n" f"args: {args}" ) - if is_open: - refnr = selected_row[0]["refnr"] - - if conn_is_ibis(self.conn): - conn = self.conn + if not is_open: + logger.debug("Raised PreventUpdate") + raise PreventUpdate + refnr = selected_row[0]["refnr"] + logger.debug(f"Trying to retrieve history for {refnr}") + connection_object = _get_connection_object() + if isinstance(connection_object, ConnectionPool): + logger.debug("Using ConnectionPool logic.") + with get_connection() as conn: t = conn.table("skjemadataendringshistorikk") df = t.filter(_.refnr == refnr).to_pandas() df["endret_tid"] = ( @@ -177,35 +176,36 @@ def historikktabell( for col in df.columns ] return df.to_dict("records"), columns - elif isinstance(self.conn, EimerDBInstance): - try: - partition_args = dict(zip(self.time_units, args, strict=False)) - df = self.conn.query_changes( - f"""SELECT * FROM {tabell} - WHERE refnr = '{refnr}' - ORDER BY datetime DESC - """, - partition_select=create_partition_select( - desired_partitions=self.time_units, - skjema=skjema, - **partition_args, - ), - ) - if df is None: - df = pd.DataFrame(columns=["ingen", "data"]) - columns = [ - { - "headerName": col, - "field": col, - "filter": True, - "resizable": True, - } - for col in df.columns - ] - return df.to_dict("records"), columns - except Exception as e: - logger.error(f"Error in historikktabell: {e}", exc_info=True) - return None, None + elif isinstance(connection_object, EimerDBInstance): + try: + partition_args = dict(zip(self.time_units, args, strict=False)) + df = connection_object.query_changes( + f"""SELECT * FROM {tabell} + WHERE refnr = '{refnr}' + ORDER BY datetime DESC + """, + partition_select=create_partition_select( + desired_partitions=self.time_units, + skjema=skjema, + **partition_args, + ), + ) + if df is None: + df = pd.DataFrame(columns=["ingen", "data"]) + columns = [ + { + "headerName": col, + "field": col, + "filter": True, + "resizable": True, + } + for col in df.columns + ] + return df.to_dict("records"), columns + except Exception as e: + logger.error(f"Error in historikktabell: {e}", exc_info=True) + return None, None else: - logger.debug("Raised PreventUpdate") - raise PreventUpdate + raise NotImplementedError( + f"Connection of type {type(connection_object)} is not currently supported." + ) diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_main_view.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_main_view.py index 2df3cef6..deca9bac 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_main_view.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_main_view.py @@ -8,10 +8,10 @@ from dash import html from dash.dependencies import Input from dash.dependencies import Output -from eimerdb import EimerDBInstance + +from ssb_dash_framework.utils import get_connection from ...setup.variableselector import VariableSelector -from ...utils.core_query_functions import conn_is_ibis from .altinn_editor_comment import AltinnEditorComment from .altinn_editor_contact import AltinnEditorContact from .altinn_editor_control import AltinnEditorControl @@ -19,7 +19,6 @@ from .altinn_editor_primary_table import AltinnEditorPrimaryTable from .altinn_editor_submitted_forms import AltinnEditorSubmittedForms from .altinn_editor_supporting_table import AltinnEditorSupportTables -from .altinn_editor_supporting_table import add_year_diff_support_table from .altinn_editor_unit_details import AltinnEditorUnitDetails from .altinn_editor_utility import AltinnEditorStateTracker @@ -45,7 +44,6 @@ class AltinnSkjemadataEditor: def __init__( self, time_units: list[str], - conn: object, starting_table: str | None = None, variable_connection: dict[str, str] | None = None, sidepanels: None = None, @@ -56,7 +54,6 @@ def __init__( Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. starting_table: Table to be selected by default in module. If None, defaults to first table it finds. variable_connection: Dict containing the name of characteristics from the dataset as keys and the variable selector name associated with it as value. sidepanels: Later might be used for customizing sidepanel modules. @@ -69,8 +66,6 @@ def __init__( self.icon = "🗊" self.label = "Data editor" - add_year_diff_support_table(conn) - self.conn = conn self.variable_connection = variable_connection if variable_connection else {} self.variableselector = VariableSelector( @@ -87,19 +82,16 @@ def __init__( if primary_view is None: self.primary_table = AltinnEditorPrimaryTable( time_units=time_units, - conn=self.conn, variable_selector_instance=self.variableselector, ) if sidepanels is None: self.sidepanels: list[AltinnEditorModule] = [ AltinnEditorSubmittedForms( time_units=time_units, - conn=self.conn, variable_selector_instance=self.variableselector, ), AltinnEditorUnitDetails( time_units=time_units, - conn=self.conn, variable_connection=self.variable_connection, variable_selector_instance=self.variableselector, ), @@ -109,22 +101,17 @@ def __init__( AltinnEditorSupportTables(), AltinnEditorContact( time_units=time_units, - conn=self.conn, variable_selector_instance=self.variableselector, ), AltinnEditorHistory( time_units=time_units, - conn=self.conn, variable_selector_instance=self.variableselector, ), AltinnEditorControl( time_units=time_units, - conn=self.conn, variable_selector_instance=self.variableselector, ), - AltinnEditorComment( - conn=self.conn, - ), + AltinnEditorComment(), ] self.is_valid() self.module_callbacks() @@ -135,22 +122,11 @@ def is_valid(self) -> None: def get_skjemadata_table_names(self) -> list[dict[str, str]]: """Retrieves the names of all the skjemadata-tables in the eimerdb.""" - if isinstance(self.conn, EimerDBInstance): - all_tables = list(self.conn.tables.keys()) + with get_connection() as conn: skjemadata_tables = [ - element for element in all_tables if element.startswith("skjemadata") + table for table in conn.list_tables() if table.startswith("skjemadata_") ] - elif conn_is_ibis(self.conn): - skjemadata_tables = [ - table - for table in self.conn.list_tables() - if table.startswith("skjemadata_") - ] - else: - raise TypeError( - f"Connection object conn supplied to 'AltinnSkjemadataEditor' is not supported. Received: {type(self.conn)}" - ) - return [{"label": item, "value": item} for item in skjemadata_tables] + return [{"label": item, "value": item} for item in skjemadata_tables] def skjemadata_table_selector(self) -> dbc.Col: """Makes a dropdown for selecting which 'skjemadata' table to view.""" diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_primary_table.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_primary_table.py index 01f5cc6a..a8f3571b 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_primary_table.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_primary_table.py @@ -2,7 +2,6 @@ from typing import Any import dash_ag_grid as dag -import ibis from dash import callback from dash import html from dash.dependencies import Input @@ -11,13 +10,15 @@ from dash.exceptions import PreventUpdate from eimerdb import EimerDBInstance from ibis import _ +from psycopg_pool import ConnectionPool -from ssb_dash_framework.utils import conn_is_ibis from ssb_dash_framework.utils import create_filter_dict from ssb_dash_framework.utils import ibis_filter_with_dict from ...setup.variableselector import VariableSelector from ...utils.alert_handler import create_alert +from ...utils.config_tools.connection import _get_connection_object +from ...utils.config_tools.connection import get_connection from ...utils.eimerdb_helpers import create_partition_select logger = logging.getLogger(__name__) @@ -33,7 +34,6 @@ class AltinnEditorPrimaryTable: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, cols_to_hide: list[str] | None = None, ) -> None: @@ -41,19 +41,12 @@ def __init__( Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. variable_selector_instance: An instance of VariableSelector for variable selection. cols_to_hide: A list of columns to ignore. Defaults to ["row_id","row_ids",*self.time_units,"skjema","refnr"]. Raises: - TypeError: If variable_selector_instance is not an instance of VariableSelector. Or - if connection object is neither EimerDBInstance or Ibis connection. + TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -118,7 +111,6 @@ def layout(self) -> html.Div: def module_callbacks(self) -> None: """Defines the callbacks for the module.""" - # check if var-bedrift exists try: self.variableselector.get_option("var-bedrift", search_target="id") @@ -171,117 +163,113 @@ def hovedside_update_altinnskjema( logger.info("Returning nothing.") logger.debug(f"Args length: {len(args)}") return [], [] - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query(f"SELECT * FROM {tabell}") - conn.create_table(tabell, data) - datatyper = self.conn.query("SELECT * FROM datatyper") - conn.create_table("datatyper", datatyper) - filter_dict = create_filter_dict( - self.time_units, [int(x) for x in args] - ) - elif conn_is_ibis(self.conn): - conn = self.conn - filter_dict = create_filter_dict(self.time_units, args) - else: - raise TypeError("Connection object is invalid type.") - columns = conn.table(tabell).columns - if "variabel" in columns and "verdi" in columns: - long_format = True - else: - long_format = False + if isinstance(_get_connection_object(), EimerDBInstance): + args = tuple([int(x) for x in args]) - if long_format: - logger.debug("Processing long data") - try: - t = conn.table(tabell) - d = conn.table("datatyper") - d = d.filter(ibis_filter_with_dict(filter_dict)) - partition_args = dict(zip(self.time_units, args, strict=False)) - logger.debug( - f"partition_select:\n{create_partition_select(desired_partitions=self.time_units,skjema=skjema,**partition_args,)}" - ) - - t = t.filter(_.refnr == refnr).join(d, "variabel", how="left") - t = t.filter(ibis_filter_with_dict(filter_dict)) - - # sort by bedrift if available - if bedrift and "ident" in t.columns: - t = t.order_by([_.ident.cases((bedrift, 0), else_=1), _.radnr]) - else: - t = t.order_by(_.radnr) - - df = t.drop( - [col for col in t.columns if col.endswith("_right")] - + ["datatype", "radnr", "tabell"] - ).to_pandas() - logger.debug(f"resultat dataframe:\n{df.head(2)}") - columndefs = [ - { - "headerName": col, - "field": col, - "hide": col - in [ - "row_id", - "row_ids", - *self.time_units, - "skjema", - "refnr", - ], - "flex": 2 if col == "variabel" else 1, - } - for col in df.columns - ] - return df.to_dict("records"), columndefs - except Exception as e: - logger.error( - f"Error in hovedside_update_altinnskjema (long format): {e}", - exc_info=True, - ) - return None, None - else: - logger.debug("Processing wide data") - try: - partition_args = dict(zip(self.time_units, args, strict=False)) - t = conn.table(tabell) - - df = ( - t.filter(ibis_filter_with_dict(filter_dict)) - .filter(_.refnr == refnr) - .to_pandas() - ) - - # sort by bedrift if it exists - if bedrift and "ident" in df.columns: - df = df.sort_values( - by="ident", - key=lambda x: x.map(lambda v: 0 if v == bedrift else 1), + filter_dict = create_filter_dict( + self.time_units, args + ) # May need args to be ints for eimerdb? + + with get_connection(necessary_tables=[tabell, "datatyper"]) as conn: + columns = conn.table(tabell).columns + if "variabel" in columns and "verdi" in columns: + long_format = True + else: + long_format = False + if long_format: + logger.debug("Processing long data") + try: + t = conn.table(tabell) + d = conn.table("datatyper") + d = d.filter(ibis_filter_with_dict(filter_dict)) + partition_args = dict(zip(self.time_units, args, strict=False)) + logger.debug( + f"partition_select:\n{create_partition_select(desired_partitions=self.time_units,skjema=skjema,**partition_args,)}" ) + t = t.filter(_.refnr == refnr).join(d, "variabel", how="left") + t = t.filter(ibis_filter_with_dict(filter_dict)) + # sort by bedrift if available + if bedrift and "ident" in t.columns: + t = t.order_by( + [_.ident.cases((bedrift, 0), else_=1), _.radnr] + ) + else: + t = t.order_by(_.radnr) + + df = t.drop( + [col for col in t.columns if col.endswith("_right")] + + ["datatype", "radnr", "tabell"] + ).to_pandas() + logger.debug(f"resultat dataframe:\n{df.head(2)}") + columndefs = [ + { + "headerName": col, + "field": col, + "hide": col + in [ + "row_id", + "row_ids", + *self.time_units, + "skjema", + "refnr", + ], + "flex": 2 if col == "variabel" else 1, + } + for col in df.columns + ] + return df.to_dict("records"), columndefs + except Exception as e: + logger.error( + f"Error in hovedside_update_altinnskjema (long format): {e}", + exc_info=True, + ) + return None, None + else: + logger.debug("Processing wide data") + try: + partition_args = dict(zip(self.time_units, args, strict=False)) + t = conn.table(tabell) - columndefs = [ - { - "headerName": col, - "field": col, - "hide": col - in [ - "row_id", - "row_ids", - "enhetsinfo_row_ids", - *self.time_units, - "skjema", - "refnr", - ], - } - for col in df.columns - ] - return df.to_dict("records"), columndefs + df = ( + t.filter(ibis_filter_with_dict(filter_dict)) + .filter(_.refnr == refnr) + .to_pandas() + ) - except Exception as e: - logger.error( - f"Error in hovedside_update_altinnskjema (wide format): {e}", - exc_info=True, - ) - return None, None + # sort by bedrift if it exists + if bedrift and "ident" in df.columns: + df = df.sort_values( + by="ident", + key=lambda x: x.map(lambda v: 0 if v == bedrift else 1), + ) + columndefs = [ + { + "headerName": col, + "field": col, + "hide": col + in [ + "row_id", + "row_ids", + "enhetsinfo_row_ids", + *self.time_units, + "skjema", + "refnr", + ], + "flex": ( + 2 if col == "variabel" else 1 + ), # What does this do actually? + } + for col in df.columns + ] + logger.debug(f"resultat dataframe:\n{df.head(2)}") + + return df.to_dict("records"), columndefs + except Exception as e: + logger.error( + f"Error in hovedside_update_altinnskjema (long format): {e}", + exc_info=True, + ) + return None, None @callback( # type: ignore[misc] Output("var-statistikkvariabel", "value"), @@ -332,60 +320,62 @@ def update_table( f"alert_store: {alert_store}\n" f"args: {args}" ) - if conn_is_ibis(self.conn): - period_where = [ - f"{x} = '{edited[0]['data'][x]}'" for x in self.time_units - ] - ident = edited[0]["data"]["ident"] - refnr = edited[0]["data"]["refnr"] - value = edited[0]["value"] - old_value = edited[0]["oldValue"] - condition_str = " AND ".join(period_where) - columns = self.conn.table(tabell).columns - if "variabel" in columns and "verdi" in columns: - try: - variable = edited[0]["data"]["variabel"] - query = f""" - UPDATE {tabell} - SET verdi = '{value}' - WHERE variabel = '{variable}' AND ident = '{ident}' AND refnr = '{refnr}' AND {condition_str} - """ - self.conn.raw_sql(query) - alert_store = [ - create_alert( - f"ident: {ident}, variabel: {variable} er oppdatert fra {old_value} til {value}!", - "success", - ephemeral=True, - ), - *alert_store, - ] - except Exception as e: - raise e - else: - try: - variable = edited[0]["colId"] - query = f""" - UPDATE {tabell} - SET {variable} = '{value}' - WHERE ident = '{ident}' AND refnr = '{refnr}' AND {condition_str} - """ - self.conn.raw_sql(query) - alert_store = [ - create_alert( - f"ident: {ident}, {variable} er oppdatert fra {old_value} til {value}!", - "success", - ephemeral=True, - ), - *alert_store, - ] - except Exception as e: - raise e - return alert_store + connection_object = _get_connection_object() + if isinstance(connection_object, ConnectionPool): + with get_connection() as conn: + period_where = [ + f"{x} = '{edited[0]['data'][x]}'" for x in self.time_units + ] + ident = edited[0]["data"]["ident"] + refnr = edited[0]["data"]["refnr"] + value = edited[0]["value"] + old_value = edited[0]["oldValue"] + condition_str = " AND ".join(period_where) + columns = conn.table(tabell).columns + if "variabel" in columns and "verdi" in columns: + try: + variable = edited[0]["data"]["variabel"] + query = f""" + UPDATE {tabell} + SET verdi = '{value}' + WHERE variabel = '{variable}' AND ident = '{ident}' AND refnr = '{refnr}' AND {condition_str} + """ + conn.raw_sql(query) + alert_store = [ + create_alert( + f"ident: {ident}, variabel: {variable} er oppdatert fra {old_value} til {value}!", + "success", + ephemeral=True, + ), + *alert_store, + ] + except Exception as e: + raise e + else: + try: + variable = edited[0]["colId"] + query = f""" + UPDATE {tabell} + SET {variable} = '{value}' + WHERE ident = '{ident}' AND refnr = '{refnr}' AND {condition_str} + """ + conn.raw_sql(query) + alert_store = [ + create_alert( + f"ident: {ident}, {variable} er oppdatert fra {old_value} til {value}!", + "success", + ephemeral=True, + ), + *alert_store, + ] + except Exception as e: + raise e + return alert_store - elif isinstance(self.conn, EimerDBInstance): + elif isinstance(connection_object, EimerDBInstance): partition_args = dict(zip(self.time_units, args, strict=False)) tables_editable_dict = {} - data_dict = self.conn.tables + data_dict = connection_object.tables for table, details in data_dict.items(): if table.startswith("skjemadata") and "schema" in details: @@ -398,7 +388,7 @@ def update_table( table_editable_dict = tables_editable_dict[tabell] edited_column = edited[0]["colId"] - schema = self.conn.tables[tabell]["schema"] + schema = connection_object.tables[tabell]["schema"] columns = {field["name"] for field in schema} if "variabel" in columns and "verdi" in columns: long_format = True @@ -412,7 +402,7 @@ def update_table( ident = edited[0]["data"]["ident"] try: - self.conn.query( + connection_object.query( f"""UPDATE {tabell} SET {edited_column} = '{new_value}' WHERE row_id = '{row_id}' @@ -464,5 +454,5 @@ def update_table( return alert_store else: raise TypeError( - f"Conection 'self.conn' is not a valid connection object. Is type: {type(self.conn)}" + f"Conection set by set_connection() is not a valid connection object. Is type: {type(connection_object)}" ) diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_submitted_forms.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_submitted_forms.py index 52e0b074..cc977297 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_submitted_forms.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_submitted_forms.py @@ -3,7 +3,6 @@ import dash_ag_grid as dag import dash_bootstrap_components as dbc -import ibis from dash import callback from dash import dcc from dash import html @@ -13,13 +12,15 @@ from dash.exceptions import PreventUpdate from eimerdb import EimerDBInstance from ibis import _ +from psycopg_pool import ConnectionPool -from ssb_dash_framework.utils import conn_is_ibis from ssb_dash_framework.utils import create_filter_dict from ssb_dash_framework.utils import ibis_filter_with_dict from ...setup.variableselector import VariableSelector +from ...utils import _get_connection_object from ...utils import create_alert +from ...utils import get_connection from ...utils.eimerdb_helpers import create_partition_select from .altinn_editor_utility import AltinnEditorStateTracker @@ -32,24 +33,17 @@ class AltinnEditorSubmittedForms: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, ) -> None: """Initializes the Altinn Editor submitted forms module. Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. variable_selector_instance: An instance of VariableSelector for variable selection. Raises: TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -162,36 +156,28 @@ def update_skjemaer( logger.debug(f"Args:\nident: {ident}\nargs: {args}") if ident is None or any(arg is None for arg in args): return [], None - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM enheter") - conn.create_table("enheter", data) - filter_dict = create_filter_dict( - self.time_units, [int(x) for x in args] - ) - elif conn_is_ibis(self.conn): - conn = self.conn - filter_dict = create_filter_dict(self.time_units, args) - else: - raise TypeError("Connection object is invalid type.") - print(filter_dict) - try: - t = conn.table("enheter") - skjemaer = ( - t.filter(ibis_filter_with_dict(filter_dict)) - .filter(_.ident == ident) - .select("skjema") - .distinct(on="skjema") - .to_pandas()["skjema"] - .to_list() - ) + if isinstance(_get_connection_object(), EimerDBInstance): + args = tuple([int(x) for x in args]) + filter_dict = create_filter_dict(self.time_units, args) - options = [{"label": item, "value": item} for item in skjemaer] - value = options[0]["value"] - return options, value - except Exception as e: - logger.error(f"Error in update_skjemaer: {e}", exc_info=True) - return [], None + with get_connection() as conn: + try: + t = conn.table("enheter") + skjemaer = ( + t.filter(ibis_filter_with_dict(filter_dict)) + .filter(_.ident == ident) + .select("skjema") + .distinct(on="skjema") + .to_pandas()["skjema"] + .to_list() + ) + + options = [{"label": item, "value": item} for item in skjemaer] + value = options[0]["value"] + return options, value + except Exception as e: + logger.error(f"Error in update_skjemaer: {e}", exc_info=True) + return [], None @callback( # type: ignore[misc] Output("alert_store", "data", allow_duplicate=True), @@ -221,73 +207,52 @@ def set_skjema_to_edited( variabel = edited[0]["colId"] new_value = edited[0]["value"] refnr = edited[0]["data"]["refnr"] + if variabel not in ["aktiv", "editert"]: + raise ValueError( + f"In the submitted forms module only 'aktiv' and 'editert' are editable fields. You tried to edit '{variabel}." + ) - if variabel == "editert": - try: - self.conn.query( - f""" - UPDATE skjemamottak - SET editert = {new_value} - WHERE refnr = '{refnr}' - """, - partition_select=create_partition_select( - desired_partitions=self.time_units, - skjema=skjema, - **partition_args, - ), - ) - return [ - create_alert( - f"Skjema {refnr} sin editeringsstatus er satt til {new_value}.", - "success", - ephemeral=True, - ), - *alert_store, - ] - except Exception: - return [ - create_alert( - "En feil skjedde under oppdatering av editeringsstatusen", - "danger", - ephemeral=True, - ), - *alert_store, - ] - elif variabel == "aktiv": - try: - self.conn.query( - f""" - UPDATE skjemamottak - SET aktiv = {new_value} - WHERE refnr = '{refnr}' - """, + query = f""" + UPDATE skjemamottak + SET {variabel} = {new_value} + WHERE refnr = '{refnr}' + """ + + connection_object = _get_connection_object() + try: + if isinstance(connection_object, ConnectionPool): + + with get_connection() as conn: + conn.raw_sql(query) + + elif isinstance(connection_object, EimerDBInstance): + connection_object.query( + query, partition_select=create_partition_select( desired_partitions=self.time_units, skjema=skjema, **partition_args, ), ) - return [ - create_alert( - f"Skjema {refnr} sin aktivstatus er satt til {new_value}.", - "success", - ephemeral=True, - ), - *alert_store, - ] - except Exception: - return [ - create_alert( - "En feil skjedde under oppdatering av aktivstatusen", - "danger", - ephemeral=True, - ), - *alert_store, - ] - else: - logger.debug(f"Tried to edit {variabel}, preventing update.") - logger.debug("Raised PreventUpdate") - raise PreventUpdate + + return [ + create_alert( + f"Skjema {refnr} sin editeringsstatus er satt til {new_value}.", + "success", + ephemeral=True, + ), + *alert_store, + ] + except Exception as e: + logger.debug(e) + return [ + create_alert( + "En feil skjedde under oppdatering av editeringsstatusen", + "danger", + ephemeral=True, + ), + *alert_store, + ] @callback( # type: ignore[misc] Output("altinnedit-table-skjemaer", "rowData"), @@ -303,38 +268,32 @@ def update_sidebar_table( print("Varselector: ", self.variableselector.states) if skjema is None or ident is None or any(arg is None for arg in args): return None, None - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM skjemamottak") - conn.create_table("skjemamottak", data) - filter_dict = create_filter_dict( - self.variableselector.states, [int(x) for x in args] - ) - elif conn_is_ibis(self.conn): - conn = self.conn - filter_dict = create_filter_dict(self.variableselector.states, args) - try: - t = conn.table("skjemamottak") - df = ( - t.filter(ibis_filter_with_dict(filter_dict)) - .filter(_.ident == ident) - .order_by(_.dato_mottatt) - .select("dato_mottatt", "skjema", "refnr", "editert", "aktiv") - .to_pandas() - ) - columns = [ - ( - {"headerName": col, "field": col, "editable": True} - if col in ["editert", "aktiv"] - else {"headerName": col, "field": col} + if isinstance(_get_connection_object(), EimerDBInstance): + args = tuple([int(x) for x in args]) + filter_dict = create_filter_dict(self.variableselector.states, args) + with get_connection(necessary_tables=["skjemamottak"]) as conn: + try: + t = conn.table("skjemamottak") + df = ( + t.filter(ibis_filter_with_dict(filter_dict)) + .filter(_.ident == ident) + .order_by(_.dato_mottatt) + .select("skjema", "dato_mottatt", "refnr", "editert", "aktiv") + .to_pandas() ) - for col in df.columns - ] - print("Test: ", df) - return df.to_dict("records"), columns - except Exception as e: - logger.error(f"Error in update_sidebar_table: {e}", exc_info=True) - return None, None + columns = [ + ( + {"headerName": col, "field": col, "editable": True} + if col in ["editert", "aktiv"] + else {"headerName": col, "field": col} + ) + for col in df.columns + ] + print("Test: ", df) + return df.to_dict("records"), columns + except Exception as e: + logger.error(f"Error in update_sidebar_table: {e}", exc_info=True) + return None, None @callback( # type: ignore[misc] Output("altinnedit-table-skjemaer", "selectedRows"), diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_supporting_table.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_supporting_table.py index 8847d317..43b3f2ec 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_supporting_table.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_supporting_table.py @@ -6,7 +6,6 @@ import dash_ag_grid as dag import dash_bootstrap_components as dbc -import ibis import pandas as pd from dash import callback from dash import html @@ -14,12 +13,11 @@ from dash.dependencies import Output from dash.dependencies import State from dash.exceptions import PreventUpdate -from eimerdb import EimerDBInstance from ibis import _ from ssb_dash_framework.setup import VariableSelector +from ssb_dash_framework.utils import get_connection -from ...utils.core_query_functions import conn_is_ibis from .altinn_editor_utility import AltinnEditorStateTracker logger = logging.getLogger(__name__) @@ -125,28 +123,20 @@ def support_table_layout(self) -> dbc.Tab: def add_year_diff_support_table( - conn: Any, + table_to_diff, ) -> None: # TODO make actually return two periods and diff. """Adds a table showing difference to previous year.""" def year_diff_support_table_get_data_func(ident: str, year: str) -> pd.DataFrame: - if conn_is_ibis(conn): - logger.info("Assuming is ibis connection.") - connection = conn - elif isinstance(conn, EimerDBInstance): - connection = ibis.polars.connect() - data = conn.query( - f"SELECT * FROM skjemadata_hoved WHERE ident = {ident} AND aar = {year}" - ) - connection.create_table("skjemadata_hoved", data) - else: - raise TypeError("Wah") # TODO fix - try: - s = connection.table("skjemadata_hoved") - except Exception: - # fallback option - s = connection.table("core_skjemadata_mapped") - return s.filter(_.ident == ident).to_pandas() + with get_connection() as conn: + try: + s = conn.table("skjemadata_hoved") + except Exception: + # fallback option + s = conn.table( + "core_skjemadata_mapped" + ) # TODO: Fix? Temporary for nøku use + return s.filter(_.ident == ident).to_pandas() AltinnSupportTable( label="Endring fra fjorår", diff --git a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_unit_details.py b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_unit_details.py index 70b41e8e..e53200c3 100644 --- a/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_unit_details.py +++ b/src/ssb_dash_framework/modules/altinn_editor/altinn_editor_unit_details.py @@ -1,10 +1,8 @@ import logging -import time from typing import Any import dash_ag_grid as dag import dash_bootstrap_components as dbc -import ibis from dash import callback from dash import html from dash.dependencies import Input @@ -14,11 +12,12 @@ from eimerdb import EimerDBInstance from ibis import _ -from ssb_dash_framework.utils import conn_is_ibis from ssb_dash_framework.utils import create_filter_dict from ssb_dash_framework.utils import ibis_filter_with_dict from ...setup.variableselector import VariableSelector +from ...utils.config_tools.connection import _get_connection_object +from ...utils.config_tools.connection import get_connection logger = logging.getLogger(__name__) @@ -29,7 +28,6 @@ class AltinnEditorUnitDetails: def __init__( self, time_units: list[str], - conn: object, variable_selector_instance: VariableSelector, variable_connection: dict[str, str], ) -> None: @@ -37,18 +35,12 @@ def __init__( Args: time_units: List of time units to be used in the module. - conn: Database connection object that must have a 'query' method. variable_selector_instance: An instance of VariableSelector for variable selection. variable_connection: Dict containing the name of characteristics from the dataset as keys and the variable selector name associated with it as value. Raises: TypeError: If variable_selector_instance is not an instance of VariableSelector. """ - if not isinstance(conn, EimerDBInstance) and not conn_is_ibis(conn): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(conn)}" - ) - self.conn = conn if not isinstance(variable_selector_instance, VariableSelector): raise TypeError( "variable_selector_instance must be an instance of VariableSelector" @@ -126,32 +118,21 @@ def update_enhetsinfotabell( args, ) return None, None - time.sleep( - 1 - ) # TODO: Fix some kind of multithreading to let it query more than one thing at a time. try: - if isinstance(self.conn, EimerDBInstance): - conn = ibis.polars.connect() - data = self.conn.query("SELECT * FROM enhetsinfo") - conn.create_table("enhetsinfo", data) - filter_dict = create_filter_dict( - self.time_units, [int(x) for x in args] + if isinstance(_get_connection_object(), EimerDBInstance): + args = tuple([int(x) for x in args]) + filter_dict = create_filter_dict(self.time_units, args) + with get_connection(necessary_tables=["enhetsinfo"]) as conn: + t = conn.table("enhetsinfo") + df = ( + t.filter(_.ident == ident) + .filter(ibis_filter_with_dict(filter_dict)) + .to_pandas() ) - elif conn_is_ibis(self.conn): - conn = self.conn - filter_dict = create_filter_dict(self.time_units, args) - else: - raise TypeError("Connection object is invalid type.") - print(filter_dict) - t = conn.table("enhetsinfo") - df = ( - t.filter(_.ident == ident) - .filter(ibis_filter_with_dict(filter_dict)) - .to_pandas() - ) - df = df.drop(columns=["row_id", "id", "foretak"], errors="ignore") - columns = [{"headerName": col, "field": col} for col in df.columns] - return df.to_dict("records"), columns + df = df.drop(columns=["row_id", "id", "foretak"], errors="ignore") + columns = [{"headerName": col, "field": col} for col in df.columns] + return df.to_dict("records"), columns + except Exception as e: logger.error(f"Error in update_enhetsinfotabell: {e}", exc_info=True) return None, None diff --git a/src/ssb_dash_framework/modules/freesearch.py b/src/ssb_dash_framework/modules/freesearch.py index cd081a5e..399fedc1 100644 --- a/src/ssb_dash_framework/modules/freesearch.py +++ b/src/ssb_dash_framework/modules/freesearch.py @@ -12,12 +12,10 @@ from dash.dependencies import Output from dash.dependencies import State from dash.exceptions import PreventUpdate -from eimerdb import EimerDBInstance - -from ssb_dash_framework.utils import conn_is_ibis from ..utils import TabImplementation from ..utils import WindowImplementation +from ..utils import get_connection from ..utils.module_validation import module_validator logger = logging.getLogger(__name__) @@ -34,7 +32,7 @@ class FreeSearch(ABC): _id_number = 0 - def __init__(self, database: Any, label: str = "Frisøk") -> None: + def __init__(self, conn=None, label: str = "Frisøk") -> None: """Initialize the FreeSearch module. Args: @@ -44,17 +42,13 @@ def __init__(self, database: Any, label: str = "Frisøk") -> None: Raises: TypeError: If the connection object is not 'EimerDBInstance' or ibis connection. """ - if not isinstance(database, EimerDBInstance) and not conn_is_ibis(database): - raise TypeError( - f"The database object must be 'EimerDBInstance' or ibis connection. Received: {type(database)}" - ) self.module_number = FreeSearch._id_number self.module_name = self.__class__.__name__ FreeSearch._id_number += 1 self.icon = "🔍" self.label = label + self.conn = conn - self.database = database self.module_layout = self._create_layout() self.module_callbacks() module_validator(self) @@ -165,8 +159,11 @@ def table_free_search( raise PreventUpdate if partition is not None: partition = ast.literal_eval(partition) - - df = self.database.query(query, partition_select=partition) + if not self.conn: + with get_connection(partition_select=partition) as conn: + df = conn.raw_sql(query) + else: + df = self.conn.raw_sql(query) columns = [ { "headerName": col, @@ -187,26 +184,26 @@ class FreeSearchTab(TabImplementation, FreeSearch): specific to the tab interface. """ - def __init__(self, database: Any) -> None: + def __init__(self, conn: Any | None = None) -> None: """Initialize the FreeSearchTab with a database connection. Args: database: Database connection or interface used for executing SQL queries. """ - FreeSearch.__init__(self, database=database) + FreeSearch.__init__(self, conn=conn) TabImplementation.__init__(self) class FreeSearchWindow(WindowImplementation, FreeSearch): """FreeSearchWindow is a class that creates a modal based on the FreeSearch module.""" - def __init__(self, database: Any) -> None: + def __init__(self, conn: Any | None = None) -> None: """Initialize the FreeSearchWindow class. Args: database: The database connection or object used for querying. """ - FreeSearch.__init__(self, database=database) + FreeSearch.__init__(self, conn=conn) WindowImplementation.__init__( self, ) diff --git a/src/ssb_dash_framework/utils/__init__.py b/src/ssb_dash_framework/utils/__init__.py index 27b0b4d7..fdc841b0 100644 --- a/src/ssb_dash_framework/utils/__init__.py +++ b/src/ssb_dash_framework/utils/__init__.py @@ -3,6 +3,12 @@ from .alert_handler import AlertHandler from .alert_handler import create_alert from .app_logger import enable_app_logging +from .config_tools import _get_connection_callable +from .config_tools import _get_connection_object +from .config_tools import get_connection +from .config_tools import set_connection +from .config_tools import set_eimerdb_connection +from .config_tools import set_postgres_connection from .core_query_functions import active_no_duplicates_refnr_list from .core_query_functions import conn_is_ibis from .core_query_functions import create_filter_dict @@ -28,6 +34,8 @@ "DemoDataCreator", "TabImplementation", "WindowImplementation", + "_get_connection_callable", + "_get_connection_object", "_get_kostra_r", "active_no_duplicates_refnr_list", "conn_is_ibis", @@ -36,9 +44,13 @@ "create_database_engine", "create_filter_dict", "enable_app_logging", + "get_connection", "hb_method", "ibis_filter_with_dict", "module_validator", + "set_connection", + "set_eimerdb_connection", + "set_postgres_connection", "sidebar_button", # "th_error", ] diff --git a/src/ssb_dash_framework/utils/config_tools/__init__.py b/src/ssb_dash_framework/utils/config_tools/__init__.py new file mode 100644 index 00000000..67bd7e48 --- /dev/null +++ b/src/ssb_dash_framework/utils/config_tools/__init__.py @@ -0,0 +1,17 @@ +"""Contains tools and utilities for configuring the app from a central point.""" + +from .connection import _get_connection_callable +from .connection import _get_connection_object +from .connection import get_connection +from .connection import set_connection +from .connection import set_eimerdb_connection +from .connection import set_postgres_connection + +__all__ = [ + "_get_connection_callable", + "_get_connection_object", + "get_connection", + "set_connection", + "set_eimerdb_connection", + "set_postgres_connection", +] diff --git a/src/ssb_dash_framework/utils/config_tools/connection.py b/src/ssb_dash_framework/utils/config_tools/connection.py new file mode 100644 index 00000000..86740598 --- /dev/null +++ b/src/ssb_dash_framework/utils/config_tools/connection.py @@ -0,0 +1,220 @@ +from collections.abc import Callable +from collections.abc import Iterator +from contextlib import AbstractContextManager +from contextlib import contextmanager +from typing import Any + +import ibis +from eimerdb import EimerDBInstance +from ibis.backends import BaseBackend +from ibis.backends.postgres import Backend +from psycopg_pool import ConnectionPool + +_IS_POOLED: bool | None = None +_CONNECTION: object | None = None +_CONNECTION_CALLABLE: Callable[..., Any] | None = None + + +def _get_connection_object() -> object | None: + """Getter function to retrieve the connection object. + + Used for retrieving the connection object the app is using as default after running 'set_connection'. + """ + global _CONNECTION + return _CONNECTION + + +def _get_connection_callable() -> Callable[..., Any] | None: + """Getter function to retrieve the connection callable. + + Used for retrieving the connection callable the app is using as default after running 'set_connection'. + """ + global _CONNECTION_CALLABLE + return _CONNECTION_CALLABLE + + +@contextmanager +def get_connection(**kwargs: Any) -> Iterator[BaseBackend]: + """Getter function to get the ibis connection object. + + Args: + **kwargs: Leaves room for connections that require keyword arguments. + + Yields: + conn: Connection object, primarily an ibis.Backend object. + + Example: + with get_connection() as conn: + t = conn.table('datatable') + t.select([*variables]).to_pandas() + + Raises: + ValueError: If no connection has been set using 'set_connection()'. + """ + global _IS_POOLED, _CONNECTION_CALLABLE + if not _CONNECTION_CALLABLE: + raise ValueError("No connection has been set.") + with _CONNECTION_CALLABLE(**kwargs) as conn: + yield conn + + +def set_connection( + connection_func: Callable[..., AbstractContextManager[BaseBackend]], + is_pooled: bool = False, +) -> None: + """Setter function to set the function for generating ibis connection objects. + + Args: + connection_func: Callable that needs to provide an ibis connection using '@contextmanager' decorator that and yielding an ibis connection. + is_pooled: Indicates if the connection method utilizes pooling. Defaults to False. + + Raises: + TypeError: If yielded connection object is not an ibis.BaseBackend object. + """ + global _IS_POOLED, _CONNECTION, _CONNECTION_CALLABLE + + _IS_POOLED = is_pooled + + with connection_func() as yielded_conn_object: + if not isinstance(yielded_conn_object, BaseBackend): + raise TypeError( + f"Currently this framework only supports connections based on ibis-framework backend objects. Received '{type(yielded_conn_object)}'" + ) + + _CONNECTION_CALLABLE = connection_func + + +def set_postgres_connection( + database_url: str, pool_min_size: int = 1, pool_max_size: int = 1 +) -> None: + """Helper function to configure a pooled connection to a postgres database. + + Args: + database_url: Connection url for the database. Gets passed to psycopg_pool.ConnectionPool as conninfo argument. + pool_min_size: The minimum size of the pool. Defaults to 1. + pool_max_size: The maximum size of the pool. Defaults to 1. + """ + global _IS_POOLED, _CONNECTION, _CONNECTION_CALLABLE + _IS_POOLED = True + + pool = ConnectionPool( + conninfo=database_url, min_size=pool_min_size, max_size=pool_max_size + ) + _CONNECTION = pool + + @contextmanager + def _wrap_ibis_postgres(*args: Any, **kwargs: Any) -> Iterator[BaseBackend]: + with pool.connection() as raw_conn: + yield Backend.from_connection(raw_conn) + + set_connection(_wrap_ibis_postgres) + + +def set_eimerdb_connection( + bucket_name: str, eimer_name: str, tables_default: list[str] | None = None +) -> None: + """Helper function to configure a ssb-dash-framework compatible connection to an eimerdb database. + + Works by reading the data from eimerdb, creating an in-memory duckdb database for the read data, and then creating an ibis connection to that in-memory duckdb database. + Could probably be optimized to reuse the same in-memory database, but as the future of eimerdb is unsure that might be a waste of time. + + Args: + bucket_name: The name of the Google Cloud Storage bucket where the EimerDB database is hosted. + eimer_name: The name of the EimerDB instance, the database name so to speak. + tables_default: Default tables to load when nothing else is specified. Some modules require a pre-set selection of tables to be loaded into the duckdb database to function properly. + This argument provides a way to affect which tables are loaded by default. + If left as None, it defaults to using 'DEFAULT_TABLES' which are 'enheter', 'kontaktinfo', 'skjemamottak', 'skjemadata_hoved' and 'datatyper' + + Raises: + ValueError: If the default tables provided is not a list of strings. + """ + global _IS_POOLED, _CONNECTION, _CONNECTION_CALLABLE + _IS_POOLED = False + _CONNECTION = EimerDBInstance( + bucket_name=bucket_name, + eimer_name=eimer_name, + ) + + DEFAULT_TABLES = [ + "enheter", + "kontaktinfo", + "skjemamottak", + "skjemadata_hoved", + "datatyper", + ] + if tables_default is None: + necessary_tables_default = DEFAULT_TABLES + else: + necessary_tables_default = tables_default + + if not isinstance(necessary_tables_default, list) or not all( + isinstance(item, str) for item in necessary_tables_default + ): + raise ValueError( + f"'tables_default' must be list[str]. Received: {necessary_tables_default!r}" + ) + + @contextmanager + def _eimer_ibis_converter( + necessary_tables: list[str] | None = None, + partition_select: None | dict[str, list[Any]] = None, + ) -> Iterator[BaseBackend]: + global _CONNECTION + conn = ibis.connect("duckdb://") + + tables_to_read = necessary_tables or necessary_tables_default + + if not isinstance(necessary_tables_default, list) or not all( + isinstance(item, str) for item in necessary_tables_default + ): + raise ValueError( + f"'necessary_tables_default' must be list[str]. " + f"Received: {necessary_tables_default!r}" + ) + + if "enheter" in tables_to_read: + enheter = _CONNECTION.query( + "SELECT * FROM enheter", + partition_select=partition_select, + ) + conn.create_table("enheter", enheter) + if "kontaktinfo" in tables_to_read: + kontaktinfo = _CONNECTION.query( + "SELECT * FROM kontaktinfo", + partition_select=partition_select, + ) + conn.create_table("kontaktinfo", kontaktinfo) + if "skjemamottak" in tables_to_read: + skjemamottak = _CONNECTION.query( + "SELECT * FROM skjemamottak", + partition_select=partition_select, + ) + conn.create_table("skjemamottak", skjemamottak) + if "skjemadata_hoved" in tables_to_read: + skjemadata = _CONNECTION.query( + "SELECT * FROM skjemadata_hoved", + partition_select=partition_select, + ) + conn.create_table("skjemadata_hoved", skjemadata) + if "datatyper" in tables_to_read: + datatyper = _CONNECTION.query( + "SELECT * FROM datatyper", + partition_select=partition_select, + ) + conn.create_table("datatyper", datatyper) + if "kontroller" in tables_to_read: + kontroller = _CONNECTION.query( + "SELECT * FROM kontroller", + partition_select=partition_select, + ) + conn.create_table("kontroller", kontroller) + if "kontrollutslag" in tables_to_read: + kontrollutslag = _CONNECTION.query( + "SELECT * FROM kontrollutslag", + partition_select=partition_select, + ) + conn.create_table("kontrollutslag", kontrollutslag) + + yield conn + + _CONNECTION_CALLABLE = _eimer_ibis_converter diff --git a/tests/conftest.py b/tests/conftest.py index fa7b6313..920a0e6d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,11 @@ from collections.abc import Generator +from contextlib import contextmanager import ibis import pytest from ssb_dash_framework import VariableSelector +from ssb_dash_framework import set_connection @pytest.fixture(autouse=True) @@ -22,6 +24,24 @@ def clear_VariableSelector_variableselectoroptions() -> Generator[None, None, No VariableSelector._variableselectoroptions.clear() +@pytest.fixture(autouse=True, scope="session") +def testing_connection(): + ibis_polars_conn = ibis.polars.connect() + + @contextmanager + def _ibis_polars_cm(*args, **kwargs): + yield ibis_polars_conn + + set_connection(_ibis_polars_cm, is_pooled=False) + + yield ibis_polars_conn + + try: + ibis_polars_conn.close() + except Exception: + pass + + @pytest.fixture(autouse=True, scope="session") def ibis_polars_conn(): print("Setting up connection...") diff --git a/tests/module_tests/test_agg_dist_plotter.py b/tests/module_tests/test_agg_dist_plotter.py index 94cef6c3..c91adc3b 100644 --- a/tests/module_tests/test_agg_dist_plotter.py +++ b/tests/module_tests/test_agg_dist_plotter.py @@ -10,7 +10,7 @@ def test_import() -> None: assert AggDistPlotterWindow is not None -def test_instantiation(ibis_polars_conn) -> None: +def test_instantiation() -> None: set_variables( [ "aar", @@ -20,5 +20,5 @@ def test_instantiation(ibis_polars_conn) -> None: "altinnskjema", # Required by the module itself ] ) - AggDistPlotterTab(time_units=["aar", "maaned"], conn=ibis_polars_conn) - AggDistPlotterWindow(time_units=["aar", "maaned"], conn=ibis_polars_conn) + AggDistPlotterTab(time_units=["aar", "maaned"]) + AggDistPlotterWindow(time_units=["aar", "maaned"]) diff --git a/tests/module_tests/test_altinn_editor.py b/tests/module_tests/test_altinn_editor.py index 8e805cce..b455c4f1 100644 --- a/tests/module_tests/test_altinn_editor.py +++ b/tests/module_tests/test_altinn_editor.py @@ -8,8 +8,6 @@ def test_import() -> None: assert AltinnSupportTable is not None -def test_instantiation(ibis_polars_conn) -> None: +def test_instantiation() -> None: set_variables(["year", "quarter", "refnr", "statistikkvariabel", "ident"]) - AltinnSkjemadataEditor( - time_units=["year", "quarter"], conn=ibis_polars_conn, variable_connection={} - ) + AltinnSkjemadataEditor(time_units=["year", "quarter"], variable_connection={}) diff --git a/tests/module_tests/test_freesearch.py b/tests/module_tests/test_freesearch.py index ce413877..83db3343 100644 --- a/tests/module_tests/test_freesearch.py +++ b/tests/module_tests/test_freesearch.py @@ -9,6 +9,11 @@ def test_import_freesearch() -> None: assert FreeSearchWindow is not None, "FreeSearchWindow is not importable" -def test_instantiation(ibis_polars_conn) -> None: - FreeSearchTab(database=ibis_polars_conn) - FreeSearchWindow(database=ibis_polars_conn) +def test_instantiation_default_connection() -> None: + FreeSearchTab() + FreeSearchWindow() + + +def test_instantiation_custom_conn(ibis_polars_conn) -> None: + FreeSearchTab(conn=ibis_polars_conn) + FreeSearchWindow(conn=ibis_polars_conn)