Merge pull request #101 from ourzora/BACK-2749

zylora · web-flow · commit e8bc81ce3d32 · 2024-04-25T21:17:23.000-04:00
BACK-2749: support svg image uri in opensea parser
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## v0.3.3
+
+- Fix an issue in `OpenseaParser` where the plain-text svg wouldn't be recognized as valid image uri
+- Add check in `DefaultCatchallParser` to require that `raw_data` be a `dict`
+
 ## v0.3.2
 
 - Fix an issue in `DataURIAdapter` where plain-text json data uri would get ignored
diff --git a/docs/index.md b/docs/index.md
@@ -1,6 +1,6 @@
 # Getting Started
 
-Documentation for version: **v0.3.2**
+Documentation for version: **v0.3.3**
 
 ## Overview
 
diff --git a/offchain/metadata/parsers/catchall/default_catchall.py b/offchain/metadata/parsers/catchall/default_catchall.py
@@ -242,4 +242,7 @@ def should_parse_token(  # type: ignore[no-untyped-def]
         Returns:
             bool: whether or not the collection parser handles this token.
         """
-        return bool(token.uri and raw_data)
+
+        if raw_data is not None and not isinstance(raw_data, dict):
+            logger.info("DefaultCatchallParser skips token {token} due to invalid raw data")
+        return bool(token.uri and raw_data is not None and isinstance(raw_data, dict))
diff --git a/offchain/metadata/parsers/collection/artblocks.py b/offchain/metadata/parsers/collection/artblocks.py
@@ -224,7 +224,7 @@ def get_additional_fields(self, raw_data: dict) -> list[MetadataField]:  # type:
 
         return additional_fields
 
-    def parse_traits(self, raw_data: dict) -> Optional[list[Attribute]]:  # type: ignore[type-arg]  # noqa: E501
+    def parse_traits(self, raw_data: dict) -> list[Attribute]:  # type: ignore[type-arg]  # noqa: E501
         traits = raw_data.get("traits")
         if not traits or not isinstance(traits, list):
             return  # type: ignore[return-value]
diff --git a/offchain/metadata/parsers/collection/ens.py b/offchain/metadata/parsers/collection/ens.py
@@ -58,7 +58,7 @@ def get_additional_fields(self, raw_data: dict) -> list[MetadataField]:  # type:
             )
         return additional_fields
 
-    def parse_attributes(self, raw_data: dict) -> Optional[list[Attribute]]:  # type: ignore[type-arg]  # noqa: E501
+    def parse_attributes(self, raw_data: dict) -> list[Attribute]:  # type: ignore[type-arg]  # noqa: E501
         attributes = raw_data.get("attributes")
         if not attributes or not isinstance(attributes, list):
             return  # type: ignore[return-value]
diff --git a/offchain/metadata/parsers/schema/opensea.py b/offchain/metadata/parsers/schema/opensea.py
@@ -1,3 +1,4 @@
+import base64
 from typing import Optional
 
 from offchain.metadata.models.metadata import (
@@ -90,6 +91,9 @@ def parse_metadata(self, token: Token, raw_data: dict, *args, **kwargs) -> Optio
         image = None
         image_uri = raw_data.get("image") or raw_data.get("image_data")
         if image_uri:
+            if image_uri.startswith("<svg"):
+                image_uri_encoded = base64.b64encode(image_uri.encode("utf-8")).decode("utf-8")
+                image_uri = f"data:image/svg+xml;base64,{image_uri_encoded}"
             image_mime, image_size = self.fetcher.fetch_mime_type_and_size(image_uri)
             image = MediaDetails(size=image_size, uri=image_uri, mime_type=image_mime)
 
@@ -140,6 +144,9 @@ async def _gen_parse_metadata_impl(self, token: Token, raw_data: dict, *args, **
         image = None
         image_uri = raw_data.get("image") or raw_data.get("image_data")
         if image_uri:
+            if image_uri.startswith("<svg"):
+                image_uri_encoded = base64.b64encode(image_uri.encode("utf-8")).decode("utf-8")
+                image_uri = f"data:image/svg+xml;base64,{image_uri_encoded}"
             image_mime, image_size = await self.fetcher.gen_fetch_mime_type_and_size(
                 image_uri
             )
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "offchain"
-version = "0.3.2"
+version = "0.3.3"
 description = "Open source metadata processing framework"
 authors = ["Zora eng <eng@zora.co>"]
 readme = "README.md"
diff --git a/tests/metadata/parsers/test_default_catchall_parser.py b/tests/metadata/parsers/test_default_catchall_parser.py
@@ -5,7 +5,7 @@
 import pytest
 
 from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher
-from offchain.metadata.models.metadata import Metadata, MetadataStandard
+from offchain.metadata.models.metadata import Metadata
 from offchain.metadata.models.token import Token
 from offchain.metadata.parsers.catchall.default_catchall import DefaultCatchallParser
 
@@ -14,7 +14,7 @@ class TestDefaultCatchallParser:
     token = Token(
         chain_identifier="ETHEREUM-MAINNET",
         collection_address="0x74cb086a1611cc9ca672f458b7742dd4159ac9db",
-        token_id="80071",
+        token_id=80071,
         uri="https://api.dego.finance/gego-token-v2/80071",
     )
     raw_data = {
diff --git a/tests/metadata/parsers/test_opensea_parser.py b/tests/metadata/parsers/test_opensea_parser.py
@@ -3,6 +3,7 @@
 from unittest.mock import MagicMock
 
 import pytest
+import base64
 
 from offchain.metadata.adapters.ipfs import IPFSAdapter
 from offchain.metadata.fetchers.metadata_fetcher import MetadataFetcher
@@ -12,7 +13,6 @@
     Metadata,
     MetadataField,
     MetadataFieldType,
-    MetadataStandard,
 )
 from offchain.metadata.models.token import Token
 from offchain.metadata.parsers.schema.opensea import OpenseaParser
@@ -22,7 +22,7 @@ class TestOpenseaParser:
     token = Token(
         chain_identifier="ETHEREUM-MAINNET",
         collection_address="0x5180db8f5c931aae63c74266b211f580155ecac8",
-        token_id="1",
+        token_id=1,
         uri="ipfs://QmSr3vdMuP2fSxWD7S26KzzBWcAN1eNhm4hk1qaR3x3vmj/1.json",
     )
 
@@ -229,3 +229,44 @@ async def test_opensea_parser_gen_parses_metadata(self, raw_crypto_coven_metadat
             token=self.token, raw_data=raw_crypto_coven_metadata
         )
         assert metadata
+
+    @pytest.mark.asyncio
+    async def test_opensea_parser_parses_token_with_xml_image(self):
+        parser = OpenseaParser(fetcher=MetadataFetcher())  # type: ignore[abstract]
+        token = Token(
+            chain_identifier="BASE-MAINNET",
+            collection_address="0x00000000001594c61dd8a6804da9ab58ed2483ce",
+            token_id=91107139416293979998100172630436458595092238971,
+            uri="https://metadata.nfts2me.com/api/ownerTokenURI/8453/91107139416293979998100172630436458595092238971/574759207385280074438303243253258373278259074888/10000/",
+        )
+        raw_data = {
+            "name": "NFTs2Me Collection Owner - drako",
+            "description": "Represents **Ownership of the NFTs2Me Collection** with address '[0x0fF562Ab42325222cF72971d32ED9CDF373b927B](https://0x0fF562Ab42325222cF72971d32ED9CDF373b927B_8453.nfts2.me/)'.\n\nTransferring this NFT implies changing the owner of the collection, as well as who will receive 100% of the profits from primary and secondary sales.\n\n[NFTs2Me](https://nfts2me.com/) is a showcase of unique digital creations from talented creators who have used the tool to generate their own NFT projects. These projects range from digital art and collectibles to gaming items and more, all with the added value of being verified on the blockchain. With a wide range of styles and themes, the [NFTs2Me](https://nfts2me.com/) tool offers something for every fan of the growing NFT space.",
+            "image_data": '<svg viewBox="0 0 499.99998 499.99998" width="500" height="500" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="background: white;"><style type="text/css">.st0{fill:#6CB7E6;}.st1{fill:#147ABF;}</style><g transform="matrix(2.002645,0,0,2.002645,40.953902,-319.12168)"><circle class="st0" cx="105.05" cy="204.58" r="8.1800003"/><path class="st0" d="M 80.05,327.28 54.21,312.9 V 253.94 L 86.54,236.99 V 356.55 L 36.4,325.86 c 0,0 -3.28,-1.71 -3.28,-6.2 0,-4.49 0,-71.78 0,-71.78 0,0 -0.57,-4.27 3.85,-6.76 4.42,-2.49 60.67,-35.39 60.67,-35.39 l 2.07,5.04 c 0,0 -53.67,31.08 -57.95,33.85 -1.78,1.15 -3.15,2.11 -3.15,4.98 0,2.98 -0.01,54.64 -0.01,67.42 0,2.51 0.22,3.4 2.27,5.19 1.9,1.64 39.16,24.28 39.16,24.28 z"/><circle class="st0" cx="103.73" cy="363.79001" r="8.1800003"/><path class="st0" d="m 128.72,241.09 25.85,14.38 v 58.96 l -32.33,16.95 V 211.82 l 50.13,30.69 c 0,0 3.28,1.71 3.28,6.2 0,4.49 0,71.78 0,71.78 0,0 0.57,4.27 -3.85,6.76 -4.41,2.49 -60.67,35.39 -60.67,35.39 l -2.07,-5.04 c 0,0 53.67,-31.08 57.95,-33.85 1.78,-1.15 3.15,-2.11 3.15,-4.98 0,-2.98 0.01,-54.64 0.01,-67.42 0,-2.51 -0.22,-3.4 -2.27,-5.19 -1.9,-1.64 -39.16,-24.28 -39.16,-24.28 z"/></g> <g transform="matrix(2.002645,0,0,2.002645,-601.56128,-329.35424)"><polygon class="st1" points="122.24,331.38 86.53,291.04 86.53,236.99 122.24,277.39" transform="translate(320.83329,5.1095282)"/></g> <path d="m 113.85595,58.83376 h 272.2881 a 36.305083,22.490145 0 0 1 36.30509,22.490149 V 418.67609 a 36.305083,22.490145 0 0 1 -36.30509,22.49015 H 113.85595 A 36.305083,22.490145 0 0 1 77.55086,418.67609 V 81.323909 A 36.305083,22.490145 0 0 1 113.85595,58.83376 Z" style="fill:none;stroke:#147ABF;stroke-width:3;stroke-opacity:1"/> <path id="text-path" d="m 109.75187,53.071033 h 280.49626 a 37.399504,23.168113 0 0 1 37.39951,23.168117 v 347.5217 a 37.399504,23.168113 0 0 1 -37.39951,23.16812 H 109.75187 A 37.399504,23.168113 0 0 1 72.35236,423.76085 V 76.23915 a 37.399504,23.168113 0 0 1 37.39951,-23.168117 z" style="fill:none;"/><text text-rendering="optimizeSpeed"><textPath startOffset="-100%" fill="black" font-family="\'Courier New\', monospace" font-size="16px" xlink:href="#text-path">COLLECTION • drako <animate additive="sum" attributeName="startOffset" from="0%" to="100%" begin="0s" dur="30s" repeatCount="indefinite"/> </textPath> <textPath startOffset="0%" fill="black" font-family="\'Courier New\', monospace" font-size="16px" xlink:href="#text-path">COLLECTION • drako <animate additive="sum" attributeName="startOffset" from="0%" to="100%" begin="0s" dur="30s" repeatCount="indefinite"/> </textPath> <textPath startOffset="50%" fill="black" font-family="\'Courier New\', monospace" font-size="16px" xlink:href="#text-path">OWNER • 0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748 <animate additive="sum" attributeName="startOffset" from="0%" to="100%" begin="0s" dur="30s" repeatCount="indefinite"/> </textPath> <textPath startOffset="-50%" fill="black" font-family="\'Courier New\', monospace" font-size="16px" xlink:href="#text-path">OWNER • 0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748 <animate additive="sum" attributeName="startOffset" from="0%" to="100%" begin="0s" dur="30s" repeatCount="indefinite"/> </textPath> </text></svg>',
+            "background_color": "#FFFFFF",
+            "attributes": [
+                {"trait_type": "Collection Name", "value": "drako"},
+                {
+                    "trait_type": "Collection Address",
+                    "value": "0x0fF562Ab42325222cF72971d32ED9CDF373b927B",
+                },
+                {
+                    "trait_type": "Owner Address",
+                    "value": "0x64Ad181f69466bD4D15076aC1d33a22c6Cc9d748",
+                },
+                {"display_type": "number", "trait_type": "Revenue", "value": 100},
+            ],
+            "external_url": "https://0x0fF562Ab42325222cF72971d32ED9CDF373b927B_8453.nfts2.me/",
+        }
+        metadata = await parser._gen_parse_metadata_impl(token=token, raw_data=raw_data)
+        svg_encoded = base64.b64encode(
+            raw_data.get("image_data").encode("utf-8")
+        ).decode("utf-8")
+        expected_image_uri = f"data:image/svg+xml;base64,{svg_encoded}"
+        assert metadata
+        assert metadata.image == MediaDetails(
+            size=3256,
+            sha256=None,
+            uri=expected_image_uri,
+            mime_type="image/svg+xml",
+        )

Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,7 @@ def get_additional_fields(self, raw_data: dict) -> list[MetadataField]: # type:`
`58`	`58`	`)`
`59`	`59`	`return additional_fields`
`60`	`60`
`61`		`- def parse_attributes(self, raw_data: dict) -> Optional[list[Attribute]]: # type: ignore[type-arg] # noqa: E501`
	`61`	`+ def parse_attributes(self, raw_data: dict) -> list[Attribute]: # type: ignore[type-arg] # noqa: E501`
`62`	`62`	`attributes = raw_data.get("attributes")`
`63`	`63`	`if not attributes or not isinstance(attributes, list):`
`64`	`64`	`return # type: ignore[return-value]`