From 7cbe626e81d541d84b61c8fdaabe6b594e71270b Mon Sep 17 00:00:00 2001 From: Daniel Sparing Date: Thu, 6 Feb 2025 17:36:19 +0100 Subject: [PATCH 01/10] don't use `%[%]sql` within Databricks --- src/sql/magic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sql/magic.py b/src/sql/magic.py index 6d8d32f70..918538d32 100644 --- a/src/sql/magic.py +++ b/src/sql/magic.py @@ -1,4 +1,5 @@ import json +import os import re from pathlib import Path @@ -769,6 +770,12 @@ def load_SqlMagic_configs(ip): def load_ipython_extension(ip): """Load the magics, this function is executed when the user runs: %load_ext sql""" + + # If running within Databricks, do not use the `sql magics` + if "DATABRICKS_RUNTIME_VERSION" in os.environ: + del SqlMagic.magics["cell"]["sql"] + del SqlMagic.magics["line"]["sql"] + sql_magic = SqlMagic(ip) _set_sql_magic(sql_magic) ip.register_magics(sql_magic) From e6da05e4068b8b7acd8fed26665284f0fd0c63b9 Mon Sep 17 00:00:00 2001 From: Daniel Sparing Date: Thu, 6 Feb 2025 17:49:00 +0100 Subject: [PATCH 02/10] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e532cb15..3b082ff93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 0.10.18dev +* [Fix] Disabled `%sql` and `%%sql` on Databricks, fixing ([#1047](https://github.com/ploomber/jupysql/issues/1047)), so that jupysql coexsits with the SQL magic of Databricks as already [advertised](https://jupysql.ploomber.io/en/latest/howto.html#use-sql-sql-in-databricks). + ## 0.10.17 (2025-01-08) * [Feature] Disable full stack trace when using spark connect ([#1011](https://github.com/ploomber/jupysql/issues/1011)) (by [@b1ackout](https://github.com/b1ackout)) From a18d94365d06f5f747b0b0d95479cf84b6b7fdea Mon Sep 17 00:00:00 2001 From: Daniel Sparing Date: Thu, 6 Feb 2025 17:50:23 +0100 Subject: [PATCH 03/10] adjust line comment --- src/sql/magic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/magic.py b/src/sql/magic.py index 918538d32..b75829d6e 100644 --- a/src/sql/magic.py +++ b/src/sql/magic.py @@ -771,7 +771,7 @@ def load_SqlMagic_configs(ip): def load_ipython_extension(ip): """Load the magics, this function is executed when the user runs: %load_ext sql""" - # If running within Databricks, do not use the `sql magics` + # If running within Databricks, do not use the sql magics if "DATABRICKS_RUNTIME_VERSION" in os.environ: del SqlMagic.magics["cell"]["sql"] del SqlMagic.magics["line"]["sql"] From 761d7ce5936731a472503f02de1d971e7fc2d789 Mon Sep 17 00:00:00 2001 From: Daniel Sparing Date: Thu, 6 Feb 2025 18:01:14 +0100 Subject: [PATCH 04/10] pkgmt format --- scripts/large-table-gen.py | 3 +-- src/sql/column_guesser.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/large-table-gen.py b/scripts/large-table-gen.py index e63b7918d..6226aae21 100644 --- a/scripts/large-table-gen.py +++ b/scripts/large-table-gen.py @@ -1,5 +1,4 @@ -"""Renter large-table-template.sql -""" +"""Renter large-table-template.sql""" from pathlib import Path from jinja2 import Template diff --git a/src/sql/column_guesser.py b/src/sql/column_guesser.py index 12fb2912a..c41daad72 100644 --- a/src/sql/column_guesser.py +++ b/src/sql/column_guesser.py @@ -7,6 +7,7 @@ class Column(list): "Store a column of tabular data; record its name and whether it is numeric" + is_quantity = True name = "" From 8db775a05a1b855e3ee079c5640430fe35024bfd Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 11:26:01 -0600 Subject: [PATCH 05/10] upgrade action --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a44b0372..30d37d2d1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -113,7 +113,7 @@ jobs: nox --session test_unit --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: failed-image-artifacts ${{ matrix.os }} ${{ matrix.python-version }} @@ -158,7 +158,7 @@ jobs: nox --session test_unit_sqlalchemy_one --no-install --reuse-existing-virtualenvs - name: Upload failed images artifacts sqlalchemyv1 - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: name: failed-image-artifacts-sqlalchemy ${{ matrix.os }} ${{ matrix.python-version }} From e6dd03795715d91c0a01289522764a3f0f6e3574 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 11:28:49 -0600 Subject: [PATCH 06/10] doc fix --- .readthedocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yml b/.readthedocs.yml index a6e6ac719..114885f42 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -21,3 +21,4 @@ conda: sphinx: builder: html fail_on_warning: true + configuration: doc/_config.yml From ba324ff7e31de3dc151cb231e65757cf05a2fef7 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 11:34:17 -0600 Subject: [PATCH 07/10] doc updates, changelog, version --- .github/workflows/ci.yaml | 3 +-- CHANGELOG.md | 4 ++-- doc/howto.md | 4 +++- src/sql/__init__.py | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 30d37d2d1..68cc66aa5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -98,9 +98,8 @@ jobs: - name: Lint run: | python -m pip install --upgrade pip - python -m pip install --upgrade pkgmt codespell nox + python -m pip install --upgrade pkgmt nox pkgmt lint - codespell - name: Install dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b082ff93..c64ef7324 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # CHANGELOG -## 0.10.18dev +## 0.11.0dev -* [Fix] Disabled `%sql` and `%%sql` on Databricks, fixing ([#1047](https://github.com/ploomber/jupysql/issues/1047)), so that jupysql coexsits with the SQL magic of Databricks as already [advertised](https://jupysql.ploomber.io/en/latest/howto.html#use-sql-sql-in-databricks). +* [API Change] Disabled `%sql` and `%%sql` on Databricks ([#1047](https://github.com/ploomber/jupysql/issues/1047)) ## 0.10.17 (2025-01-08) diff --git a/doc/howto.md b/doc/howto.md index bc8fe04b8..462b04094 100644 --- a/doc/howto.md +++ b/doc/howto.md @@ -361,7 +361,9 @@ some_engine = create_engine("sqlite:///some.db") ## Use `%sql`/`%%sql` in Databricks -Databricks uses the same name (`%sql`/`%%sql`) for its SQL magics; however, JupySQL exposes a `%jupysql`/`%%jupysql` alias so you can use both: +Databricks uses the same name (`%sql`/`%%sql`) for its SQL magics; however, since this +clashes with Databricks' `%%sql` command, JupySQL will enable an `%jupysql`/`%%jupysql` +alias if a `DATABRICKS_RUNTIME_VERSION` is declared: ```{code-cell} ipython3 %jupysql duckdb:// diff --git a/src/sql/__init__.py b/src/sql/__init__.py index 0f7b37951..562481125 100644 --- a/src/sql/__init__.py +++ b/src/sql/__init__.py @@ -1,7 +1,7 @@ from sql.magic import load_ipython_extension -__version__ = "0.10.18dev" +__version__ = "0.11.0dev" __all__ = ["load_ipython_extension"] From dcfbce8620868f2cdb343491c67e53aa308bde88 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 12:25:57 -0600 Subject: [PATCH 08/10] add test --- src/sql/magic.py | 2 +- src/tests/test_magic.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sql/magic.py b/src/sql/magic.py index b75829d6e..c8e7a7eb5 100644 --- a/src/sql/magic.py +++ b/src/sql/magic.py @@ -705,7 +705,7 @@ def get_query_type(command: str): def set_configs(ip, file_path, alternate_path): """Set user defined SqlMagic configuration settings""" - sql = ip.find_cell_magic("sql").__self__ + sql = ip.find_cell_magic("jupysql").__self__ user_configs, loaded_from = util.get_user_configs(file_path, alternate_path) default_configs = util.get_default_configs(sql) table_rows = [] diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 0b0c0af93..4ee2b9fb8 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -23,6 +23,8 @@ from sql.run.resultset import ResultSet from sql import magic from sql.warnings import JupySQLQuotedNamedParametersWarning +from sql._testing import TestingShell +from sql.magic import load_ipython_extension from conftest import runsql @@ -2767,3 +2769,21 @@ def test_disabled_named_parameters_shows_disabled_warning(ip): ) assert expected_warning in str(excinfo.value) + + +def test_databricks_sql_magic_disabled(monkeypatch): + monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "10.0.0") + ip = TestingShell.preconfigured_shell() + load_ipython_extension(ip) + + ip.run_cell("%jupysql duckdb://") + ip.run_cell("%jupysql select 1") + + with pytest.raises(UsageError) as excinfo_line: + ip.run_cell("%sql select 1") + + with pytest.raises(UsageError) as excinfo_cell: + ip.run_cell("%%sql\nselect 1") + + assert "Line magic function `%sql` not found" in str(excinfo_line.value) + assert "Cell magic `%%sql` not found" in str(excinfo_cell.value) From 8716cc7c8090871be4826a807792009e74473f7c Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 12:36:17 -0600 Subject: [PATCH 09/10] skip test --- src/tests/test_magic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 4ee2b9fb8..20e5c6575 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -2771,6 +2771,9 @@ def test_disabled_named_parameters_shows_disabled_warning(ip): assert expected_warning in str(excinfo.value) +@pytest.mark.skip( + reason="Running this breaks all subsequent tests because TestingShell is a singleton. We need to find a way to isolate this test" +) def test_databricks_sql_magic_disabled(monkeypatch): monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "10.0.0") ip = TestingShell.preconfigured_shell() From 139179688e1bcb4d04f2f1ccc85fec56875ef336 Mon Sep 17 00:00:00 2001 From: Eduardo Blancas Date: Sun, 9 Feb 2025 12:36:41 -0600 Subject: [PATCH 10/10] lint --- src/tests/test_magic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/test_magic.py b/src/tests/test_magic.py index 20e5c6575..8295441a4 100644 --- a/src/tests/test_magic.py +++ b/src/tests/test_magic.py @@ -2772,7 +2772,10 @@ def test_disabled_named_parameters_shows_disabled_warning(ip): @pytest.mark.skip( - reason="Running this breaks all subsequent tests because TestingShell is a singleton. We need to find a way to isolate this test" + reason=( + "Running this breaks all subsequent tests because " + "TestingShell is a singleton. We need to find a way to isolate this test" + ) ) def test_databricks_sql_magic_disabled(monkeypatch): monkeypatch.setenv("DATABRICKS_RUNTIME_VERSION", "10.0.0")