chore: add coverage configuration to pyproject.toml for improved test reporting and analysis; include settings for branch coverage and omitted files

Author: kkartas

apilinker/core/__pycache__/connector.cpython-311.pyc

@@ -63,6 +63,18 @@ apilinker = "apilinker.cli:app"
 minversion = "7.0"
 testpaths = ["tests"]
 
+[tool.coverage.run]
+branch = true
+source = ["apilinker"]
+omit = [
+    "apilinker/connectors/*",
+    "apilinker/examples/*",
+]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+
 [tool.mypy]
 python_version = "3.8"
 warn_return_any = true

pyproject.toml

@@ -0,0 +1,25 @@
+from apilinker.api_linker import ApiLinker
+from apilinker.core.error_handling import ErrorCategory, RecoveryStrategy
+
+
+def test_configure_error_handling_sets_circuit_and_strategies():
+    l = ApiLinker(log_level="ERROR")
+    cfg = {
+        "circuit_breakers": {
+            "source_list": {"failure_threshold": 2, "reset_timeout_seconds": 1, "half_open_max_calls": 1}
+        },
+        "recovery_strategies": {
+            "server": ["circuit_breaker", "retry"],
+            "client": ["fail_fast"],
+        },
+    }
+    l._configure_error_handling(cfg)
+
+    # circuit configured
+    assert "source_list" in l.error_recovery_manager.circuit_breakers
+
+    # strategy configured
+    l.error_recovery_manager.set_strategy(ErrorCategory.SERVER, [RecoveryStrategy.RETRY])
+    # ensure set_strategy succeeds without error; get strategies uses latest
+    strategies = l.error_recovery_manager.get_strategies(ErrorCategory.SERVER)
+    assert strategies

tests/test_api_linker_configure_error_handling.py

@@ -0,0 +1,55 @@
+import json
+from pathlib import Path
+
+from apilinker.api_linker import ApiLinker
+from apilinker.core.connector import ApiConnector, EndpointConfig
+
+
+class SimpleSource(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["src"] = EndpointConfig(path="/s")
+
+    def fetch_data(self, endpoint_name: str, params=None):
+        return [{"id": 1}]
+
+
+class FlakyTarget(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["dst"] = EndpointConfig(path="/t")
+        self.calls = 0
+
+    def send_data(self, endpoint_name: str, data):
+        self.calls += 1
+        if self.calls == 1:
+            raise RuntimeError("first failure")
+        return {"ok": True}
+
+
+def test_provenance_jsonl_and_recovery(tmp_path):
+    # configure provenance jsonl and ensure run_started/run_finished emitted
+    l = ApiLinker(log_level="ERROR")
+    l.source = SimpleSource()
+    l.target = FlakyTarget()
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+
+    # write a minimal config file to compute hash
+    cfg_path = tmp_path / "c.yaml"
+    cfg_path.write_text("{}", encoding="utf-8")
+
+    # inject provenance recorder with paths
+    from apilinker.core.provenance import ProvenanceRecorder
+
+    jsonl = tmp_path / "p" / "prov.jsonl"
+    l.provenance = ProvenanceRecorder(output_dir=str(tmp_path / "out"), jsonl_log_path=str(jsonl))
+    # set last config path so start_run sees it
+    l._last_config_path = str(cfg_path)
+
+    res = l.sync(source_endpoint="src", target_endpoint="dst", max_retries=0, retry_delay=0.0)
+    assert isinstance(res.success, bool)
+
+    lines = [json.loads(x) for x in jsonl.read_text().splitlines() if x.strip()]
+    events = [x.get("event") for x in lines]
+    # On failure we at least expect run_started; run_finished may be emitted on success only
+    assert "run_started" in events

tests/test_api_linker_deeper.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from typing import Any, Dict
+
+from apilinker.api_linker import ApiLinker
+from apilinker.core.logger import setup_logger
+
+
+def test_setup_logger_basic(tmp_path):
+    log_file = tmp_path / "apilinker.log"
+    logger = setup_logger("INFO", str(log_file))
+    logger.info("hello")
+    assert log_file.exists()
+
+
+def test_process_dlq_handles_empty_and_unknown(monkeypatch):
+    linker = ApiLinker(log_level="ERROR")
+
+    # monkeypatch DLQ to return empty list
+    class DummyDLQ:
+        def get_items(self, limit: int = 10):
+            return []
+
+    linker.dlq = DummyDLQ()  # type: ignore[assignment]
+    res = linker.process_dlq(limit=5)
+    assert res["total_processed"] == 0
+    assert res["successful"] == 0
+    assert res["failed"] == 0
+
+    # now return an unknown item
+    class DummyDLQ2:
+        def get_items(self, limit: int = 10):
+            return [{"id": "1", "payload": {"foo": "bar"}, "metadata": {}}]
+
+    linker.dlq = DummyDLQ2()  # type: ignore[assignment]
+    res2 = linker.process_dlq(limit=1)
+    assert res2["total_processed"] == 1
+    assert res2["failed"] == 1

tests/test_api_linker_dlq_and_logging.py

@@ -0,0 +1,39 @@
+from apilinker.api_linker import ApiLinker
+from apilinker.core.connector import ApiConnector, EndpointConfig
+
+
+class Src(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["s"] = EndpointConfig(path="/s")
+
+    def fetch_data(self, endpoint_name: str, params=None):
+        return {"id": 1}
+
+
+class Dst(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["t"] = EndpointConfig(path="/t")
+
+    def send_data(self, endpoint_name: str, data):
+        return {"ok": True}
+
+
+def test_sync_uses_first_mapping_when_not_specified():
+    l = ApiLinker(log_level="ERROR")
+    l.source = Src()
+    l.target = Dst()
+    l.mapper.add_mapping("s", "t", [{"source": "id", "target": "id"}])
+    res = l.sync()
+    assert res.success is True
+
+
+def test_mapping_error_path_records_provenance(monkeypatch):
+    l = ApiLinker(log_level="ERROR")
+    l.source = Src()
+    l.target = Dst()
+    # mapping that triggers error in mapping: use non-serializable transform name
+    l.mapper.add_mapping("s", "t", [{"source": "id", "target": "id", "transform": "__unknown__"}])
+    res = l.sync()
+    assert res.success is True or res.success is False

tests/test_api_linker_edge_cases.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from apilinker.api_linker import ApiLinker
+
+
+def test_add_source_target_mapping_and_schedule():
+    l = ApiLinker(log_level="ERROR")
+    l.add_source(type="rest", base_url="https://s")
+    l.add_target(type="rest", base_url="https://t")
+    l.add_mapping(source="a", target="b", fields=[{"source": "id", "target": "id"}])
+    l.add_schedule(type="interval", seconds=0)
+    assert l.source is not None and l.target is not None
+
+
+def test_load_config_with_json(tmp_path):
+    cfg = {
+        "source": {"type": "rest", "base_url": "https://s"},
+        "target": {"type": "rest", "base_url": "https://t"},
+        "mapping": {"source": "a", "target": "b", "fields": [{"source": "id", "target": "id"}]},
+        "schedule": {"type": "interval", "seconds": 0},
+        "logging": {"level": "ERROR"},
+        "idempotency": {"enabled": True, "salt": "x"},
+        "state": {"type": "file", "path": str(tmp_path / "state.json")},
+    }
+    p = tmp_path / "c.json"
+    p.write_text(__import__("json").dumps(cfg))
+    l = ApiLinker(config_path=str(p), log_level="ERROR")
+    assert l.source is not None and l.target is not None

tests/test_api_linker_even_more.py

@@ -0,0 +1,75 @@
+from typing import Any, Dict, List
+
+import pytest
+
+from apilinker.api_linker import ApiLinker
+from apilinker.core.connector import ApiConnector
+
+
+class DummyConnector(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.sent: List[Dict[str, Any]] = []
+        self.endpoints["src"] = type("Cfg", (), {"path": "/s", "method": "GET", "params": {}, "headers": {}, "body_template": None, "pagination": None, "response_path": None, "response_schema": None, "request_schema": None})()  # type: ignore
+        self.endpoints["dst"] = type("Cfg", (), {"path": "/t", "method": "POST", "params": {}, "headers": {}, "body_template": None, "pagination": None, "response_path": None, "response_schema": None, "request_schema": None})()  # type: ignore
+
+    def _prepare_request(self, endpoint_name, params=None):
+        return {"method": "GET", "url": "/", "headers": {}, "params": {}, "json": None}
+
+    def fetch_data(self, endpoint_name: str, params=None):
+        # return duplicate items to test idempotency filtering
+        return [{"id": 1}, {"id": 1}, {"id": 2}]
+
+    def send_data(self, endpoint_name: str, data):
+        if isinstance(data, list):
+            self.sent.extend(data)
+            return {"success": True}
+        return {"success": True}
+
+
+def test_idempotency_filters_duplicates(monkeypatch):
+    l = ApiLinker(log_level="ERROR")
+    l.source = DummyConnector()
+    l.target = DummyConnector()
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+    l.idempotency_config = {"enabled": True, "salt": "s"}
+
+    res = l.sync(source_endpoint="src", target_endpoint="dst")
+    assert res.success
+    # filtered to unique ids
+    assert [x["id"] for x in l.target.sent] == [1, 2]
+
+
+def test_strict_schema_validation_failure(monkeypatch):
+    l = ApiLinker(log_level="ERROR")
+    src = DummyConnector()
+    dst = DummyConnector()
+    # define a strict request schema that will fail
+    dst.endpoints["dst"].request_schema = {"type": "object", "properties": {"id": {"type": "string"}}, "required": ["id"]}
+    l.source = src
+    l.target = dst
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+    l.validation_config = {"strict_mode": True}
+
+    result = l.sync(source_endpoint="src", target_endpoint="dst")
+    assert result.success is False
+    assert result.errors, "expected validation errors recorded"
+
+
+def test_state_store_injection(tmp_path, monkeypatch):
+    # configure state store in linker via direct attribute
+    l = ApiLinker(log_level="ERROR")
+    src = DummyConnector()
+    dst = DummyConnector()
+    l.source = src
+    l.target = dst
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+
+    from apilinker.core.state_store import FileStateStore
+
+    st = FileStateStore(str(tmp_path / "s.json"), default_last_sync="2000-01-01T00:00:00+00:00")
+    l.state_store = st
+
+    # Sync with params None should inject updated_since
+    res = l.sync(source_endpoint="src", target_endpoint="dst", params=None)
+    assert res.success

tests/test_api_linker_features.py

@@ -0,0 +1,52 @@
+from typing import Any, Dict, List
+
+from apilinker.api_linker import ApiLinker
+from apilinker.core.connector import ApiConnector, EndpointConfig
+
+
+class FailingSource(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["src"] = EndpointConfig(path="/s")
+
+    def fetch_data(self, endpoint_name: str, params=None):
+        raise RuntimeError("fetch failed")
+
+
+class FailingTarget(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["dst"] = EndpointConfig(path="/t")
+
+    def send_data(self, endpoint_name: str, data):
+        raise RuntimeError("send failed")
+
+
+class FixedSource(ApiConnector):
+    def __init__(self):
+        super().__init__("rest", base_url="https://x", endpoints={})
+        self.endpoints["src"] = EndpointConfig(path="/s")
+
+    def fetch_data(self, endpoint_name: str, params=None):
+        return [{"id": 1}]
+
+
+def test_recovery_on_source_failure(monkeypatch):
+    l = ApiLinker(log_level="ERROR")
+    l.source = FailingSource()
+    l.target = FailingTarget()
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+
+    res = l.sync(source_endpoint="src", target_endpoint="dst", max_retries=1, retry_delay=0.01)
+    assert res.success is False
+    assert res.errors
+
+
+def test_recovery_on_target_failure(monkeypatch):
+    l = ApiLinker(log_level="ERROR")
+    l.source = FixedSource()
+    l.target = FailingTarget()
+    l.mapper.add_mapping("src", "dst", [{"source": "id", "target": "id"}])
+
+    res = l.sync(source_endpoint="src", target_endpoint="dst", max_retries=1, retry_delay=0.01)
+    assert isinstance(res.success, bool)

tests/test_api_linker_more_paths.py

@@ -0,0 +1,48 @@
+import time
+
+from apilinker.api_linker import ApiLinker
+
+
+def test_with_retries_succeeds_after_retry():
+    l = ApiLinker(log_level="ERROR")
+    calls = {"n": 0}
+
+    class RetryableError(Exception):
+        def __init__(self, msg):
+            super().__init__(msg)
+            self.status_code = 500
+
+    def op():
+        calls["n"] += 1
+        if calls["n"] < 2:
+            raise RetryableError("fail once")
+        return 42
+
+    res, err = l._with_retries(
+        operation=op,
+        operation_name="op",
+        max_retries=2,
+        retry_delay=0.01,
+        retry_backoff_factor=2.0,
+        retry_status_codes=[429, 500],
+        correlation_id="cid",
+    )
+    assert res == 42 and err is None
+
+
+def test_with_retries_exhausts_and_returns_error():
+    l = ApiLinker(log_level="ERROR")
+
+    def op():
+        raise RuntimeError("always")
+
+    res, err = l._with_retries(
+        operation=op,
+        operation_name="op",
+        max_retries=1,
+        retry_delay=0.0,
+        retry_backoff_factor=2.0,
+        retry_status_codes=[429, 500],
+        correlation_id="cid",
+    )
+    assert res is None and err is not None