Skip to content

Commit d607902

Browse files
zylorazylora
committed
BACK-1791: ensure prop IPFS gateway gets used instead of pinata
1 parent 199fec0 commit d607902

File tree

9 files changed

+284
-140
lines changed

9 files changed

+284
-140
lines changed

‎offchain/metadata/adapters/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from .arweave import ARWeaveAdapter
2-
from .base_adapter import AdapterConfig, BaseAdapter
2+
from .base_adapter import Adapter, AdapterConfig, BaseAdapter
33
from .data_uri import DataURIAdapter
4+
from .default_adapter_configs import DEFAULT_ADAPTER_CONFIGS
45
from .http_adapter import HTTPAdapter
56
from .ipfs import IPFSAdapter

‎offchain/metadata/adapters/data_uri.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ def send(self, request: PreparedRequest, *args, **kwargs): # type: ignore[no-un
6262
newResponse.status_code = 200
6363
newResponse.headers = response.headers
6464
newResponse.raw = response
65-
newResponse.encoding = "utf-8"
65+
newResponse._content = response.read()
66+
newResponse.encoding = response.info().get_param("charset") or "utf-8"
6667
self.response = response
6768
finally:
6869
return newResponse

‎offchain/metadata/adapters/default_adapter_configs.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
from offchain.metadata.adapters.arweave import ARWeaveAdapter
2+
from offchain.metadata.adapters.base_adapter import AdapterConfig
3+
from offchain.metadata.adapters.data_uri import DataURIAdapter
4+
from offchain.metadata.adapters.http_adapter import HTTPAdapter
5+
from offchain.metadata.adapters.ipfs import IPFSAdapter
6+
7+
DEFAULT_ADAPTER_CONFIGS: list[AdapterConfig] = [
8+
AdapterConfig(
9+
adapter_cls=ARWeaveAdapter,
10+
mount_prefixes=["ar://"],
11+
host_prefixes=["https://arweave.net/"],
12+
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
13+
),
14+
AdapterConfig(adapter_cls=DataURIAdapter, mount_prefixes=["data:"]),
15+
AdapterConfig(
16+
adapter_cls=IPFSAdapter,
17+
mount_prefixes=[
18+
"ipfs://",
19+
"https://gateway.pinata.cloud/",
20+
"https://ipfs.io/",
21+
],
22+
host_prefixes=["https://gateway.pinata.cloud/ipfs/"],
23+
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
24+
),
25+
AdapterConfig(
26+
adapter_cls=HTTPAdapter,
27+
mount_prefixes=["https://", "http://"],
28+
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
29+
),
30+
]

‎offchain/metadata/fetchers/base_fetcher.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ def set_max_retries(self, new_max_retries: int): # type: ignore[no-untyped-def]
4040
pass
4141

4242
def register_adapter(self, adapter: Adapter, url_prefix: str): # type: ignore[no-untyped-def] # noqa: E501
43-
"""Register an adapter to a url prefix.
43+
"""Register an adapter to a url prefix. Note this only affects synchronous http
44+
requests (via the requests library).
4445
4546
Args:
4647
adapter (Adapter): an Adapter instance to register.

‎offchain/metadata/fetchers/metadata_fetcher.py‎

Lines changed: 34 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import requests
66

77
from offchain.logger.logging import logger
8-
from offchain.metadata.adapters.base_adapter import Adapter, AdapterConfig
8+
from offchain.metadata.adapters import Adapter, AdapterConfig, DEFAULT_ADAPTER_CONFIGS
99
from offchain.metadata.fetchers.base_fetcher import BaseFetcher
1010
from offchain.metadata.registries.fetcher_registry import FetcherRegistry
1111

@@ -24,7 +24,7 @@ def __init__(
2424
self,
2525
timeout: int = 30,
2626
max_retries: int = 0,
27-
async_adapter_configs: Optional[list[AdapterConfig]] = None,
27+
async_adapter_configs: Optional[list[AdapterConfig]] = DEFAULT_ADAPTER_CONFIGS,
2828
) -> None:
2929
self.timeout = timeout
3030
self.max_retries = max_retries
@@ -33,7 +33,8 @@ def __init__(
3333
self.async_adapter_configs = async_adapter_configs
3434

3535
def register_adapter(self, adapter: Adapter, url_prefix: str): # type: ignore[no-untyped-def] # noqa: E501
36-
"""Register an adapter to a url prefix.
36+
"""Register an adapter to a url prefix. Note this only affects synchronous http
37+
requests (via the requests library).
3738
3839
Args:
3940
adapter (Adapter): an Adapter instance to register.
@@ -57,35 +58,44 @@ def set_timeout(self, timeout: int): # type: ignore[no-untyped-def]
5758
"""
5859
self.timeout = timeout
5960

61+
def _get_async_adapter_for_uri(self, uri: str) -> Optional[Adapter]:
62+
if self.async_adapter_configs is None:
63+
logger.error("Async adapter config doesn't exist. This shouldn't happen!")
64+
return None
65+
66+
for async_adapter_config in self.async_adapter_configs:
67+
if any(
68+
uri.startswith(prefix) for prefix in async_adapter_config.mount_prefixes
69+
):
70+
logger.debug(
71+
f"Selected {async_adapter_config.adapter_cls.__name__} for making async http requests for uri={uri}" # noqa: E501
72+
)
73+
return async_adapter_config.adapter_cls(
74+
host_prefixes=async_adapter_config.host_prefixes,
75+
**async_adapter_config.kwargs,
76+
)
77+
logger.warning(
78+
f"Unable to selected an adapter for async http requests for uri={uri}"
79+
)
80+
return None
81+
6082
def _head(self, uri: str): # type: ignore[no-untyped-def]
6183
return self.sess.head(uri, timeout=self.timeout, allow_redirects=True)
6284

6385
def _get(self, uri: str): # type: ignore[no-untyped-def]
6486
return self.sess.get(uri, timeout=self.timeout, allow_redirects=True)
6587

6688
async def _gen(self, uri: str, method: Optional[str] = "GET") -> httpx.Response:
67-
from offchain.metadata.pipelines.metadata_pipeline import (
68-
DEFAULT_ADAPTER_CONFIGS,
69-
)
70-
71-
configs = DEFAULT_ADAPTER_CONFIGS
72-
73-
if self.async_adapter_configs:
74-
configs = self.async_adapter_configs
75-
76-
for adapter_config in configs:
77-
if any(uri.startswith(prefix) for prefix in adapter_config.mount_prefixes):
78-
adapter = adapter_config.adapter_cls(
79-
host_prefixes=adapter_config.host_prefixes, **adapter_config.kwargs
89+
async_adapter = self._get_async_adapter_for_uri(uri)
90+
if async_adapter is not None:
91+
if method == "HEAD":
92+
return await async_adapter.gen_head(
93+
url=uri, timeout=self.timeout, sess=self.async_sess
94+
)
95+
else:
96+
return await async_adapter.gen_send(
97+
url=uri, timeout=self.timeout, sess=self.async_sess
8098
)
81-
if method == "HEAD":
82-
return await adapter.gen_head(
83-
url=uri, timeout=self.timeout, sess=self.async_sess
84-
)
85-
else:
86-
return await adapter.gen_send(
87-
url=uri, timeout=self.timeout, sess=self.async_sess
88-
)
8999
return await self.async_sess.get(
90100
uri, timeout=self.timeout, follow_redirects=True
91101
)

‎offchain/metadata/pipelines/metadata_pipeline.py‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from offchain.metadata.registries.parser_registry import ParserRegistry
2424
from offchain.web3.contract_caller import ContractCaller
2525

26+
# TODO(luke): move the data repo's usage of this symbol to the new file, then remove this
2627
DEFAULT_ADAPTER_CONFIGS: list[AdapterConfig] = [
2728
AdapterConfig(
2829
adapter_cls=ARWeaveAdapter,
@@ -31,11 +32,6 @@
3132
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
3233
),
3334
AdapterConfig(adapter_cls=DataURIAdapter, mount_prefixes=["data:"]),
34-
AdapterConfig(
35-
adapter_cls=HTTPAdapter,
36-
mount_prefixes=["https://", "http://"],
37-
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
38-
),
3935
AdapterConfig(
4036
adapter_cls=IPFSAdapter,
4137
mount_prefixes=[
@@ -46,6 +42,11 @@
4642
host_prefixes=["https://gateway.pinata.cloud/ipfs/"],
4743
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
4844
),
45+
AdapterConfig(
46+
adapter_cls=HTTPAdapter,
47+
mount_prefixes=["https://", "http://"],
48+
kwargs={"pool_connections": 100, "pool_maxsize": 1000, "max_retries": 0},
49+
),
4950
]
5051

5152
DEFAULT_PARSERS = (
@@ -66,7 +67,7 @@ class MetadataPipeline(BasePipeline):
6667
mime type, and size by making network requests.
6768
parsers (list[BaseParser], optional): a list of parser instances for parsing token metadata.
6869
adapter_configs: (list[AdapterConfig], optional): a list of adapter configs used to register adapters
69-
to specified url prefixes.
70+
to specified url prefixes. This configuration affects both sync and async requests.
7071
""" # noqa: E501
7172

7273
def __init__(
@@ -79,6 +80,10 @@ def __init__(
7980
self.contract_caller = contract_caller or ContractCaller()
8081
self.fetcher = fetcher or MetadataFetcher(async_adapter_configs=adapter_configs)
8182
if adapter_configs is None:
83+
# TODO(luke): move the line below to the file's import section once this
84+
# file's DEFAULT_ADAPTER_CONFIGS is gone
85+
from offchain.metadata.adapters import DEFAULT_ADAPTER_CONFIGS
86+
8287
adapter_configs = DEFAULT_ADAPTER_CONFIGS
8388
for adapter_config in adapter_configs:
8489
self.mount_adapter(

‎offchain/utils/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ async def wrapped(*args, **kwargs): # type: ignore[no-untyped-def]
3333
logger.error(msg)
3434
if not silent:
3535
raise
36-
logger.warn(msg)
36+
logger.warning(msg)
3737
await asyncio.sleep(retry_delay)
3838
return None
3939

‎tests/metadata/fetchers/test_metadata_fetcher.py‎

Lines changed: 137 additions & 100 deletions
Large diffs are not rendered by default.

‎tests/metadata/pipelines/test_metadata_pipeline.py‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
# flake8: noqa: E501
22

3+
from pytest_httpx import HTTPXMock
34
from typing import Tuple
45
from unittest.mock import AsyncMock, MagicMock
56

67
import pytest
78

8-
from offchain.metadata.adapters.http_adapter import HTTPAdapter
9-
from offchain.metadata.adapters.ipfs import IPFSAdapter
9+
from offchain.metadata.adapters import (
10+
AdapterConfig,
11+
DEFAULT_ADAPTER_CONFIGS,
12+
HTTPAdapter,
13+
IPFSAdapter,
14+
)
1015
from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher
1116
from offchain.metadata.models.metadata import (
1217
Attribute,
@@ -18,10 +23,8 @@
1823
)
1924
from offchain.metadata.models.metadata_processing_error import MetadataProcessingError
2025
from offchain.metadata.models.token import Token
21-
from offchain.metadata.pipelines.metadata_pipeline import ( # type: ignore[attr-defined]
22-
AdapterConfig,
23-
MetadataPipeline,
24-
)
26+
from offchain.metadata.pipelines.metadata_pipeline import MetadataPipeline
27+
2528
from offchain.web3.contract_caller import ContractCaller
2629
from offchain.web3.jsonrpc import EthereumJSONRPC
2730

@@ -59,6 +62,62 @@ def test_metadata_pipeline_mounts_adapters(self): # type: ignore[no-untyped-def
5962
== ipfs_adapter
6063
)
6164

65+
@pytest.mark.asyncio
66+
async def test_ipfs_adapter_uses_specified_ipfs_provider(
67+
self, httpx_mock: HTTPXMock
68+
):
69+
# integration test, the following setup reflects usage in prod
70+
IPFS_PROVIDER = "https://ipfs.decentralized-content.com/ipfs/"
71+
72+
def set_async_adapters() -> list[AdapterConfig]:
73+
async_adapters = []
74+
for adapter in DEFAULT_ADAPTER_CONFIGS:
75+
if adapter.adapter_cls is IPFSAdapter:
76+
ipfs_adapter = AdapterConfig(
77+
adapter_cls=IPFSAdapter,
78+
mount_prefixes=[
79+
"ipfs://",
80+
"https://gateway.pinata.cloud/",
81+
"https://ipfs.io/",
82+
"https://ipfs.decentralized-content.com/",
83+
],
84+
host_prefixes=[IPFS_PROVIDER],
85+
)
86+
async_adapters.append(ipfs_adapter)
87+
88+
else:
89+
async_adapters.append(adapter)
90+
91+
return async_adapters
92+
93+
adapters = set_async_adapters()
94+
pipeline = MetadataPipeline(adapter_configs=adapters)
95+
96+
httpx_mock.add_response(
97+
json=[
98+
{
99+
"name": "Beast #485",
100+
"image": "https://gateway.pinata.cloud/ipfs/QmcimtwbWGKXLJ3pTMRu2ncEeeuK9DUwYye6uhJhZC9C6A/beast485.png",
101+
"external_url": "https://tierzeronft.com/",
102+
"attributes": [
103+
{"trait_type": "Background", "value": "Blue"},
104+
{"trait_type": "Fur", "value": "Dark Grey"},
105+
{"trait_type": "Shoes", "value": "Feet"},
106+
{"trait_type": "Eyes", "value": "Green"},
107+
{"trait_type": "Hat", "value": "Headset"},
108+
{"trait_type": "Unit", "value": "Unit I"},
109+
],
110+
}
111+
],
112+
url=f"{IPFS_PROVIDER}QmY3Lz7DfQPtPkK4n5StZcqc2zA6cmJC7wcAgzYXvGQLGm/485",
113+
)
114+
content = await pipeline.fetcher.gen_fetch_content(
115+
"https://gateway.pinata.cloud/ipfs/QmY3Lz7DfQPtPkK4n5StZcqc2zA6cmJC7wcAgzYXvGQLGm/485"
116+
)
117+
assert (
118+
content is not None
119+
), "Call to gateway.pinata.cloud did not get redirected to ipfs.decentralized-content.com"
120+
62121
def test_metadata_pipeline_fetch_token_uri(self, raw_crypto_coven_metadata): # type: ignore[no-untyped-def]
63122
token = Token(
64123
chain_identifier="ETHEREUM-MAINNET",

0 commit comments

Comments
 (0)