Merge pull request #5 from dhruvan2006/the-block

Add support for The Block

Author: dhruvan2006

README.md

@@ -35,6 +35,7 @@ handles the heavy lifting so you can focus on insights.
 - [Bitcoin Magazine Pro](https://www.bitcoinmagazinepro.com)
 - [Blockchain.com](https://www.blockchain.com/explorer/charts)
 - [Glassnode](https://studio.glassnode.com/charts/)
+- [The Block](https://www.theblock.co/data/)
 
 ## Installation
 To install the `chaindl` package, use pip:

chaindl/download.py

@@ -44,6 +44,7 @@ def download(url, start=None, end=None, **kwargs):
     BITCOINMAGAZINEPRO_BASE_URL = "https://www.bitcoinmagazinepro.com"
     BLOCKCHAIN_BASE_URL = "https://www.blockchain.com/explorer/charts"
     GLASSNODE_BASE_URL = "https://studio.glassnode.com/charts"
+    THEBLOCK_BASE_URL = "https://www.theblock.co"
 
     data = pd.DataFrame()
 
@@ -63,6 +64,8 @@ def download(url, start=None, end=None, **kwargs):
         data = scraper.blockchain._download(url, **kwargs)
     elif url.startswith(GLASSNODE_BASE_URL):
         data = scraper.glassnode._download(url, **kwargs)
+    elif url.startswith(THEBLOCK_BASE_URL):
+        data = scraper.theblock._download(url)
     else:
         raise ValueError("Unsupported source. Find the list of supported websites here: https://chaindl.readthedocs.io/")
 

chaindl/scraper/__init__.py

@@ -6,3 +6,4 @@
 from .bitcoinmagazinepro import _download
 from .blockchain import _download
 from .glassnode import _download
+from .theblock import _download

chaindl/scraper/theblock.py

@@ -0,0 +1,28 @@
+import json
+import requests
+import pandas as pd
+from urllib.parse import urlparse
+
+def _download(url):
+    parsed = urlparse(url)
+    parts = [p for p in parsed.path.split("/") if p]
+    last_three = "/".join(parts[-3:])
+
+    api = f"https://www.theblock.co/api/charts/chart/{last_three}"
+    response = requests.get(api)
+    response.raise_for_status()
+    _json = response.json()
+    jsonFile = _json.get("jsonFile", {}).get("data", {})
+    jsonFile = json.loads(jsonFile)
+
+    df = pd.concat(
+        {
+            series: pd.DataFrame(data["Data"]).set_index("Timestamp")["Result"]
+            for series, data in jsonFile["Series"].items()
+        },
+        axis=1
+    )
+    df.index = pd.to_datetime(df.index, unit="s")
+    df.index.name = "Date"
+
+    return df

docs/index.rst

@@ -33,6 +33,7 @@ It supports:
 - `Bitcoin Magazine Pro <https://www.bitcoinmagazinepro.com>`__
 - `Blockchain.com <https://www.blockchain.com/explorer/charts/>`__
 - `Glassnode <https://studio.glassnode.com/charts/>`__
+- `The Block <https://www.theblock.co/data/>`__
 
 ---
 
@@ -191,6 +192,18 @@ Example:
     url = "https://studio.glassnode.com/charts/addresses.ActiveCount?a=BTC"
     df = chaindl.download(url)
 
+The BLock (`theblock.co/data <https://www.theblock.co/data/>`__)
+---------------------------------------------------------------------------
+
+Click 'Share' and 'Copy Link' to get the URL of the respective metric.
+
+Example:
+
+.. code-block:: python
+
+    url = "https://www.theblock.co/data/crypto-markets/spot/total-exchange-volume-daily"
+    df = chaindl.download(url)
+
 Optional Arguments
 ==================
 

tests/test_theblock.py

@@ -0,0 +1,8 @@
+from chaindl.scraper.theblock import _download
+
+def test_transactions_on_the_bitcoin_network():
+    url = "https://www.theblock.co/data/on-chain-metrics/bitcoin/transactions-on-the-bitcoin-network-daily"
+    df = _download(url)
+    assert "7DMA" in df.columns
+    assert "Date" == df.index.name
+    assert not df.empty