Merge pull request #4 from dhruvan2006/glassnode

Add support for glassnode

Author: dhruvan2006

README.md

@@ -34,6 +34,7 @@ handles the heavy lifting so you can focus on insights.
 - [Bitbo Charts](https://charts.bitbo.io/)
 - [Bitcoin Magazine Pro](https://www.bitcoinmagazinepro.com)
 - [Blockchain.com](https://www.blockchain.com/explorer/charts)
+- [Glassnode](https://studio.glassnode.com/charts/)
 
 ## Installation
 To install the `chaindl` package, use pip:

chaindl/download.py

@@ -43,6 +43,7 @@ def download(url, start=None, end=None, **kwargs):
     CRYPTOQUANT_BASE_URL = "https://cryptoquant.com"
     BITCOINMAGAZINEPRO_BASE_URL = "https://www.bitcoinmagazinepro.com"
     BLOCKCHAIN_BASE_URL = "https://www.blockchain.com/explorer/charts"
+    GLASSNODE_BASE_URL = "https://studio.glassnode.com/charts"
 
     data = pd.DataFrame()
 
@@ -60,6 +61,8 @@ def download(url, start=None, end=None, **kwargs):
         data = scraper.bitcoinmagazinepro._download(url, **kwargs)
     elif url.startswith(BLOCKCHAIN_BASE_URL):
         data = scraper.blockchain._download(url, **kwargs)
+    elif url.startswith(GLASSNODE_BASE_URL):
+        data = scraper.glassnode._download(url, **kwargs)
     else:
         raise ValueError("Unsupported source. Find the list of supported websites here: https://chaindl.readthedocs.io/")
 

chaindl/scraper/__init__.py

@@ -4,4 +4,5 @@
 from .woocharts import _download
 from .cryptoquant import _download
 from .bitcoinmagazinepro import _download
-from .blockchain import _download
+from .blockchain import _download
+from .glassnode import _download

chaindl/scraper/glassnode.py

@@ -0,0 +1,89 @@
+import re
+from urllib.parse import urlparse, parse_qs
+import requests
+import pandas as pd
+
+SESSION = requests.Session()
+
+def _parse_metric_path(url: str):
+    """Convert Glassnode chart URL path into API metric path and asset."""
+    o = urlparse(url)
+    path = o.path.replace(".", "/")
+    query_params = parse_qs(o.query)
+    asset = query_params.get("a", ["BTC"])[0]
+
+    try:
+        _, charts, *prefix_parts, last_segment = path.split("/")
+    except ValueError:
+        raise ValueError(f"Invalid Glassnode URL path: {path}")
+
+    # Step 1: handle letter number boundaries: Min1Count -> Min_1Count
+    number_adjusted = re.sub(r'([a-zA-Z])([0-9])', r'\1_\2', last_segment)
+    # Step 2: handle Min_1Count -> Min_1_Count
+    snake_case = re.sub(r'(?<!^)(?=[A-Z])', '_', number_adjusted).lower()
+    new_path = "/".join(prefix_parts + [snake_case])
+    return new_path, asset, snake_case
+
+def _fetch_json(url: str):
+    """Fetch JSON from a URL with session handling and error checking."""
+    resp = SESSION.get(url)
+    resp.raise_for_status()
+    try:
+        return resp.json()
+    except ValueError:
+        raise ValueError(f"Failed to parse JSON from {url}: {resp.text}")
+
+def _process_metric_json(json_data, column_name: str):
+    """Convert Glassnode metric JSON into a clean DataFrame."""
+    df = pd.DataFrame(json_data)
+
+    if 'o' in df.columns:
+        # Expand nested object
+        columns = list(df['o'].iloc[0].keys())
+        expanded_df = pd.DataFrame([row['o'] for row in json_data])
+        expanded_df['Date'] = df['t']
+        cols = ['Date'] + columns
+        expanded_df = expanded_df[cols]
+        expanded_df.index = pd.to_datetime(expanded_df['Date'], unit='s')
+        expanded_df = expanded_df.drop(columns=['Date'])
+        return expanded_df
+    else:
+        df.rename(columns={'t': 'Date', 'v': column_name}, inplace=True)
+        df.index = pd.to_datetime(df['Date'], unit='s')
+        df = df.drop(columns=['Date'])
+        return df
+
+def _download(url: str, include_price: bool = True) -> pd.DataFrame:
+    try:
+        metric_path, asset, snake_case = _parse_metric_path(url)
+
+        # Fetch cookies
+        SESSION.get(url)
+
+        # Fetch metric
+        api_url = f"https://api.glassnode.com/v1/metrics/{metric_path}?a={asset}&referrer=charts"
+        json_data = _fetch_json(api_url)
+
+        if 'requiredPlan' in json_data:
+            raise ValueError(f"Metric {snake_case} is not available on the free plan.")
+
+        df = _process_metric_json(json_data, snake_case)
+
+        # Optionally fetch USD price
+        if include_price:
+            price_api = f"https://api.glassnode.com/v1/metrics/market/price_usd_close?a={asset}&referrer=charts"
+            json_price = _fetch_json(price_api)
+            df_price = pd.DataFrame(json_price)
+            df_price.rename(columns={'t': 'Date', 'v': 'price_usd_close'}, inplace=True)
+            df_price.index = pd.to_datetime(df_price['Date'], unit='s')
+            df_price = df_price.drop(columns=['Date'])
+            df = df.join(df_price, how='outer')
+
+        return df
+
+    except requests.exceptions.RequestException as e:
+        raise RuntimeError(f"Network error while fetching Glassnode metric: {e}")
+    except ValueError as e:
+        raise RuntimeError(f"Data error: {e}")
+    except Exception as e:
+        raise RuntimeError(f"Unexpected error: {e}")

docs/index.rst

@@ -32,6 +32,7 @@ It supports:
 - `Bitbo Charts <https://charts.bitbo.io/index/>`__
 - `Bitcoin Magazine Pro <https://www.bitcoinmagazinepro.com>`__
 - `Blockchain.com <https://www.blockchain.com/explorer/charts/>`__
+- `Glassnode <https://studio.glassnode.com/charts/>`__
 
 ---
 
@@ -177,6 +178,19 @@ Example:
 
 ---
 
+Glassnode (`studio.glassnode.com <https://studio.glassnode.com/charts/>`__)
+---------------------------------------------------------------------------
+
+Only basic (T1) metrics are supported. You can find a list of the basic metrics
+on their `website <https://docs.glassnode.com/data/metric-catalog#basic-metrics>`__.
+
+Example:
+
+.. code-block:: python
+
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCount?a=BTC"
+    df = chaindl.download(url)
+
 Optional Arguments
 ==================
 

tests/test_glassnode.py

@@ -0,0 +1,92 @@
+import pytest
+import pandas as pd
+from unittest.mock import patch, Mock
+from chaindl.scraper import glassnode
+
+def test_parse_metric_path_basic():
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCountWithContracts?a=ETH"
+    path, asset, snake_case = glassnode._parse_metric_path(url)
+    assert asset == "ETH"
+    assert "active_count_with_contracts" in snake_case
+    assert path.endswith(snake_case)
+
+def test_parse_metric_path_default_asset():
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCountWithContracts"
+    path, asset, snake_case = glassnode._parse_metric_path(url)
+    assert asset == "BTC"
+
+
+def test_process_metric_json_simple():
+    json_data = [{"t": 1680000000, "v": 42}, {"t": 1680000600, "v": 43}]
+    df = glassnode._process_metric_json(json_data, "metric_name")
+    assert isinstance(df, pd.DataFrame)
+    assert "metric_name" in df.columns
+    assert df.index.name is 'Date'
+
+def test_process_metric_json_nested():
+    json_data = [{"t": 1680000000, "o": {"a": 1, "b": 2}}, {"t": 1680000600, "o": {"a": 3, "b": 4}}]
+    df = glassnode._process_metric_json(json_data, "ignored")
+    assert isinstance(df, pd.DataFrame)
+    assert "a" in df.columns and "b" in df.columns
+    assert df.shape == (2, 2)  # Only a and b remain
+
+@patch("chaindl.scraper.glassnode.SESSION.get")
+def test_fetch_json_success(mock_get):
+    mock_resp = Mock()
+    mock_resp.raise_for_status.return_value = None
+    mock_resp.json.return_value = {"key": "value"}
+    mock_get.return_value = mock_resp
+    data = glassnode._fetch_json("https://fakeurl.com")
+    assert data["key"] == "value"
+
+@patch("chaindl.scraper.glassnode.SESSION.get")
+def test_download_includes_price(mock_get):
+    # Mock metric response
+    metric_resp = Mock()
+    metric_resp.raise_for_status.return_value = None
+    metric_resp.json.return_value = [{"t": 1680000000, "v": 100}]
+    # Mock price response
+    price_resp = Mock()
+    price_resp.raise_for_status.return_value = None
+    price_resp.json.return_value = [{"t": 1680000000, "v": 50000}]
+
+    mock_get.side_effect = [Mock(), metric_resp, price_resp]  # First GET for cookies
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCountWithContracts?a=ETH"
+
+    df = glassnode._download(url)
+    assert "price_usd_close" in df.columns
+    assert "active_count_with_contracts" in df.columns
+    assert isinstance(df, pd.DataFrame)
+
+@patch("chaindl.scraper.glassnode.SESSION.get")
+def test_download_raises_on_required_plan(mock_get):
+    resp = Mock()
+    resp.raise_for_status.return_value = None
+    resp.json.return_value = {"requiredPlan": True}
+    mock_get.side_effect = [Mock(), resp]
+
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCountWithContracts?a=ETH"
+    with pytest.raises(RuntimeError, match="Data error"):
+        glassnode._download(url)
+
+# Integration tests
+def test_download_active_count_btc():
+    url = "https://studio.glassnode.com/charts/addresses.ActiveCount?a=BTC"
+    df = glassnode._download(url)
+    assert "active_count" in df.columns
+    assert "price_usd_close" in df.columns
+    assert not df.empty
+
+def test_download_stock_to_flow_btc():
+    url = "https://studio.glassnode.com/charts/indicators.StockToFlowRatio?a=BTC"
+    df = glassnode._download(url)
+    assert "daysTillHalving" in df.columns
+    assert "price" in df.columns
+    assert "ratio" in df.columns
+    assert "price_usd_close" in df.columns
+    assert not df.empty
+
+def test_download_invalid_url():
+    url = "https://studio.glassnode.com/charts/addresses.NotARealMetric?a=BTC"
+    with pytest.raises(RuntimeError):
+        glassnode._download(url)