Fix linting, tests, and CI configuration

.circleci/config.yml

@@ -85,10 +85,10 @@ jobs:
     docker:
       - image: circleci/python:3.9
         environment:
-          TENTACLIO__CONN__POSTGRES_TEST: postgresql://:@localhost:5432/tentaclio-db
+          TENTACLIO__CONN__POSTGRES_TEST: postgresql://circleci@localhost:5432/tentaclio-db
           PIPENV_VENV_IN_PROJECT: true
 
-      - image: circleci/postgres:9.5.10
+      - image: cimg/postgres:15.10
         environment:
           # CircleCI img run under `circleci` user
           POSTGRES_USER: circleci

Pipfile

@@ -4,17 +4,14 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-# Install project as editable
 tentaclio-postgres = {editable = true,path = "."}
 pytest = "*"
 pytest-cov = "*"
 pytest-mock = "*"
-
 black = "*"
 flake8 = "*"
 isort = "*"
 mypy = "*"
-
 twine = "*"
 
 [packages]

README.md

@@ -19,3 +19,75 @@ format                        Run black and isort
 test                          Run unit tests
 circleci                      Validate circleci configuration (needs circleci cli)
 ```
+
+## Development
+
+### Setting up your environment
+
+1. Install dependencies:
+```bash
+make install
+```
+
+2. (Optional) Start PostgreSQL for functional tests:
+```bash
+docker-compose up -d
+```
+
+### Running Tests
+
+#### Option 1: Using Docker (Recommended)
+
+This repository includes a Docker Compose configuration that makes it easy to run tests locally without needing to install and configure PostgreSQL manually.
+
+**Start PostgreSQL:**
+```bash
+docker-compose up -d
+```
+
+This starts a PostgreSQL 15 container with:
+- User: `tentaclio`
+- Database: `tentaclio-db`
+- Password: `testpassword`
+- Port: `5432`
+
+**Run the test suites:**
+```bash
+# Linting
+make lint
+
+# Unit tests (don't require database)
+make unit
+
+# Functional tests (require PostgreSQL)
+TENTACLIO__CONN__POSTGRES_TEST=postgresql://tentaclio:testpassword@localhost:5432/tentaclio-db make functional
+```
+
+**Stop PostgreSQL:**
+```bash
+docker-compose down
+```
+
+#### Option 2: Using your own PostgreSQL installation
+
+If you prefer to use your own PostgreSQL instance:
+
+1. Ensure PostgreSQL is running and accessible
+2. Create a database and user for testing
+3. Set the connection string environment variable:
+```bash
+export TENTACLIO__CONN__POSTGRES_TEST=postgresql://user:password@localhost:5432/dbname
+```
+
+4. Run the tests:
+```bash
+make lint
+make unit
+make functional
+```
+
+### Test Types
+
+- **`make lint`** - Runs flake8 and mypy checks (no database required)
+- **`make unit`** - Runs unit tests with mocked dependencies (no database required)
+- **`make functional`** - Runs functional tests against a real PostgreSQL database (requires `TENTACLIO__CONN__POSTGRES_TEST` environment variable)

docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: tentaclio
+      POSTGRES_DB: tentaclio-db
+      POSTGRES_PASSWORD: testpassword
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U tentaclio -d tentaclio-db"]
+      interval: 2s
+      timeout: 5s
+      retries: 10

pyproject.toml

@@ -1,6 +1,5 @@
 [tool.black]
 line-length = 99
-skip-numeric-underscore-normalization = "True"
 target_version = ['py37']
 include = '\.pyi?$'
 exclude = '(\.mypy_cache)'

setup.cfg

@@ -2,10 +2,10 @@
 universal = 1
 
 [flake8]
-exclude = logs, .*/*.py, *.egg-info 
+exclude = logs, .*/*.py, *.egg-info
 max-line-length = 99
 max-complexity = 10
-ignore = F405,W391,W503
+ignore = E704,F405,W391,W503
 
 [mypy]
 incremental = True

src/tentaclio_postgres/__init__.py

@@ -1,4 +1,5 @@
-"""This package implements the tentaclio postgres client """
+"""This package implements the tentaclio postgres client"""
+
 from tentaclio import *  # noqa
 
 from .clients.postgres_client import PostgresClient

src/tentaclio_postgres/clients/postgres_client.py

@@ -5,7 +5,7 @@
 
 import io
 import os
-from typing import Sequence
+from typing import Optional, Sequence
 
 import pandas as pd
 from tentaclio import protocols
@@ -36,7 +36,9 @@ def _extract_url_params(self):
 
     # Postgres Copy Expert methods:
     @decorators.check_conn
-    def get_df_unsafe(self, sql_query: str, params: dict = None, **kwargs) -> pd.DataFrame:
+    def get_df_unsafe(
+        self, sql_query: str, params: Optional[dict] = None, **kwargs
+    ) -> pd.DataFrame:
         """Run a raw SQL query and return a data framem using COPY.
         Params:
             sql_query: query to execute

src/tentaclio_postgres/streams/postgres_handler.py

@@ -1,4 +1,5 @@
 """Handler for postgresql:// urls."""
+
 from typing import Tuple
 
 from tentaclio.protocols import ReaderClosable, WriterClosable

tests/conftest.py

@@ -3,12 +3,15 @@
 
 https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins
 """
+
 import io
 import os
 from typing import Sequence
 
 import pytest
-from tentaclio import URL, Reader, Writer, clients
+from tentaclio import URL, Reader, Writer
+
+import tentaclio_postgres
 
 
 POSTGRES_TEST_URL = os.getenv("TENTACLIO__CONN__POSTGRES_TEST")
@@ -36,7 +39,7 @@ def postgres_url():
 @pytest.fixture(scope="session")
 def db_client(postgres_url):
     """Create and tear down the session-wide SQLAlchemy Db connection"""
-    with clients.PostgresClient(postgres_url) as client:
+    with tentaclio_postgres.PostgresClient(postgres_url) as client:
         yield client
 
 
@@ -49,19 +52,17 @@ def application_name():
 def db_client_application_name(postgres_url, application_name, monkeypatch):
     """Create and tear down the session-wide SQLAlchemy Db connection"""
     monkeypatch.setenv("TENTACLIO__PG_APPLICATION_NAME", application_name)
-    with clients.PostgresClient(postgres_url) as client:
+    with tentaclio_postgres.PostgresClient(postgres_url) as client:
         yield client
 
 
 # Handler fixtures
 
 
 class FakeHandler(object):
-    def open_reader_for(self, url: "URL", extras: dict) -> Reader:
-        ...
+    def open_reader_for(self, url: "URL", extras: dict) -> Reader: ...
 
-    def open_writer_for(self, url: "URL", extras: dict) -> Writer:
-        ...
+    def open_writer_for(self, url: "URL", extras: dict) -> Writer: ...
 
 
 @pytest.fixture

tests/functional/postgres/test_api.py

@@ -25,8 +25,10 @@ def fixture_client(db_client):
         sqla.Column(TEST_COLUMNS[2], sqla.Integer),
     )
     db_client.set_schema(test_meta)
+    db_client.conn.commit()  # Commit so schema is visible to other connections
     yield db_client
     db_client.delete_schema(test_meta)
+    db_client.conn.commit()
 
 
 @pytest.fixture

tests/functional/postgres/test_client.py

@@ -20,8 +20,10 @@ def _client(db_client):
         sqla.Column(TEST_COLUMNS[2], sqla.Float),
     )
     db_client.set_schema(test_meta)
+    db_client.conn.commit()  # Commit so schema is visible to other connections
     yield db_client
     db_client.delete_schema(test_meta)
+    db_client.conn.commit()
 
 
 @pytest.fixture

tests/unit/streams/test_postgres_handler.py

@@ -19,7 +19,7 @@ def csv_data():
 
 
 def test_dump_csv(csv_data, csv_dumper, mocker):
-    mock = mocker.patch("tentaclio.clients.postgres_client.PostgresClient")
+    mock = mocker.patch("tentaclio_postgres.clients.postgres_client.PostgresClient")
     recorder = csv_dumper
     mock.return_value = recorder
 
@@ -33,7 +33,7 @@ def test_dump_csv(csv_data, csv_dumper, mocker):
 
 
 def test_create_client_correct_url(csv_data, csv_dumper, mocker):
-    mock = mocker.patch("tentaclio.clients.postgres_client.PostgresClient")
+    mock = mocker.patch("tentaclio_postgres.clients.postgres_client.PostgresClient")
     recorder = csv_dumper
     mock.return_value = recorder
 
@@ -44,7 +44,7 @@ def test_create_client_correct_url(csv_data, csv_dumper, mocker):
 
 
 def test_dump_csv_no_table(csv_data, csv_dumper, mocker):
-    mock = mocker.patch("tentaclio.clients.postgres_client.PostgresClient")
+    mock = mocker.patch("tentaclio_postgres.clients.postgres_client.PostgresClient")
     recorder = csv_dumper
     mock.return_value = recorder
     with pytest.raises(ValueError):