Switch the custom attrs implementation to a dependency.

scrapy_zyte_api/providers.py

@@ -26,6 +26,9 @@
     AutoProductListPage,
     AutoProductNavigationPage,
     AutoProductPage,
+    CustomAttributes,
+    CustomAttributesMetadata,
+    CustomAttributesValues,
     Item,
     JobPosting,
     Product,
@@ -76,6 +79,8 @@ class ZyteApiProvider(PageObjectInputProvider):
         ArticleNavigation,
         BrowserHtml,
         BrowserResponse,
+        CustomAttributes,
+        CustomAttributesValues,
         Geolocation,
         JobPosting,
         Product,
@@ -186,6 +191,17 @@ async def __call__(  # noqa: C901
                         }
                     )
                 continue
+            if cls_stripped in {CustomAttributes, CustomAttributesValues}:
+                zyte_api_meta["customAttributes"] = {
+                    k: (
+                        dict(v)
+                        if isinstance(v, frozenset)
+                        else list(v) if isinstance(v, tuple) else v
+                    )
+                    for k, v in cls.__metadata__[0]  # type: ignore[attr-defined]
+                }
+
+                continue
             kw = _ITEM_KEYWORDS.get(cls_stripped)
             if not kw:
                 continue
@@ -322,14 +338,32 @@ async def __call__(  # noqa: C901
                 result = AnnotatedInstance(Actions(actions_result), cls.__metadata__)  # type: ignore[attr-defined]
                 results.append(result)
                 continue
+            if cls_stripped is CustomAttributes and is_typing_annotated(cls):
+                custom_attrs_result = api_response.raw_api_response["customAttributes"]
+                result = AnnotatedInstance(
+                    CustomAttributes(
+                        CustomAttributesValues(custom_attrs_result["values"]),
+                        CustomAttributesMetadata.from_dict(
+                            custom_attrs_result["metadata"]
+                        ),
+                    ),
+                    cls.__metadata__,  # type: ignore[attr-defined]
+                )
+                results.append(result)
+                continue
+            if cls_stripped is CustomAttributesValues and is_typing_annotated(cls):
+                custom_attrs_result = api_response.raw_api_response["customAttributes"]
+                result = AnnotatedInstance(
+                    CustomAttributesValues(custom_attrs_result["values"]),
+                    cls.__metadata__,  # type: ignore[attr-defined]
+                )
+                results.append(result)
+                continue
             kw = _ITEM_KEYWORDS.get(cls_stripped)
             if not kw:
                 continue
             assert issubclass(cls_stripped, Item)
             result = cls_stripped.from_dict(api_response.raw_api_response[kw])  # type: ignore[attr-defined]
-            custom_attrs = api_response.raw_api_response.get("customAttributes")
-            if custom_attrs:
-                result.customAttributes = custom_attrs.get("values", {})  # type: ignore[attr-defined]
             if is_typing_annotated(cls):
                 result = AnnotatedInstance(result, cls.__metadata__)  # type: ignore[attr-defined]
             results.append(result)

setup.py

@@ -33,8 +33,8 @@ def get_version():
             "andi>=0.6.0",
             "scrapy-poet>=0.22.3",
             "web-poet>=0.17.0",
-            # https://github.com/zytedata/zyte-common-items/pull/100
-            "zyte-common-items @ git+https://github.com/zytedata/zyte-common-items.git@custom-attrs",
+            # https://github.com/zytedata/zyte-common-items/pull/106
+            "zyte-common-items @ git+https://github.com/zytedata/zyte-common-items.git@custom-attrs-dep",
         ]
     },
     classifiers=[

tests/mockserver.py

@@ -233,7 +233,7 @@ def render_POST(self, request):
             if "customAttributes" in request_data:
                 response_data["customAttributes"] = {
                     "metadata": {
-                        "totalInputTokens": "1000",
+                        "textInputTokens": 1000,
                     },
                     "values": {
                         "attr1": "foo",

tests/test_providers.py

@@ -3,6 +3,8 @@
 
 import pytest
 
+from scrapy_zyte_api._annotations import make_hashable
+
 pytest.importorskip("scrapy_poet")
 
 import attrs
@@ -24,7 +26,14 @@
     handle_urls,
 )
 from web_poet.pages import get_item_cls
-from zyte_common_items import AutoProductPage, BasePage, BaseProductPage, Product
+from zyte_common_items import (
+    AutoProductPage,
+    BasePage,
+    BaseProductPage,
+    CustomAttributes,
+    CustomAttributesValues,
+    Product,
+)
 from zyte_common_items.fields import auto_field
 
 from scrapy_zyte_api import Actions, ExtractFrom, Geolocation, Screenshot, actions
@@ -394,31 +403,105 @@ def parse_(self, response: DummyResponse, page: GeoProductPage):  # type: ignore
     assert "Geolocation dependencies must be annotated" in caplog.text
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
+)
 @ensureDeferred
 async def test_provider_custom_attrs(mockserver):
+    from typing import Annotated
+
+    @attrs.define
+    class CustomAttrsPage(BasePage):
+        product: Product
+        custom_attrs: Annotated[
+            CustomAttributes,
+            make_hashable(
+                {
+                    "attr1": {"type": "string", "description": "descr1"},
+                    "attr2": {"type": "number", "description": "descr2"},
+                }
+            ),
+        ]
+
+    class CustomAttrsZyteAPISpider(ZyteAPISpider):
+        def parse_(self, response: DummyResponse, page: CustomAttrsPage):  # type: ignore[override]
+            yield {
+                "product": page.product,
+                "custom_attrs": page.custom_attrs,
+            }
+
     settings = create_scrapy_settings()
     settings["ZYTE_API_URL"] = mockserver.urljoin("/")
     settings["SCRAPY_POET_PROVIDERS"] = {ZyteApiProvider: 0}
-    settings["ZYTE_API_PROVIDER_PARAMS"] = {
-        "customAttributes": {
-            "attr1": {"type": "string", "description": "descr1"},
-            "attr2": {"type": "number", "description": "descr2"},
-        }
-    }
 
-    item, url, _ = await crawl_single_item(ZyteAPISpider, HtmlResource, settings)
+    item, url, _ = await crawl_single_item(
+        CustomAttrsZyteAPISpider, HtmlResource, settings
+    )
     assert item["product"] == Product.from_dict(
         dict(
             url=url,
             name="Product name",
             price="10",
             currency="USD",
-            customAttributes={
+        )
+    )
+    assert item["custom_attrs"] == CustomAttributes.from_dict(
+        {
+            "values": {
                 "attr1": "foo",
                 "attr2": 42,
             },
+            "metadata": {"textInputTokens": 1000},
+        }
+    )
+
+
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
+)
+@ensureDeferred
+async def test_provider_custom_attrs_values(mockserver):
+    from typing import Annotated
+
+    @attrs.define
+    class CustomAttrsPage(BasePage):
+        product: Product
+        custom_attrs: Annotated[
+            CustomAttributesValues,
+            make_hashable(
+                {
+                    "attr1": {"type": "string", "description": "descr1"},
+                    "attr2": {"type": "number", "description": "descr2"},
+                }
+            ),
+        ]
+
+    class CustomAttrsZyteAPISpider(ZyteAPISpider):
+        def parse_(self, response: DummyResponse, page: CustomAttrsPage):  # type: ignore[override]
+            yield {
+                "product": page.product,
+                "custom_attrs": page.custom_attrs,
+            }
+
+    settings = create_scrapy_settings()
+    settings["ZYTE_API_URL"] = mockserver.urljoin("/")
+    settings["SCRAPY_POET_PROVIDERS"] = {ZyteApiProvider: 0}
+
+    item, url, _ = await crawl_single_item(
+        CustomAttrsZyteAPISpider, HtmlResource, settings
+    )
+    assert item["product"] == Product.from_dict(
+        dict(
+            url=url,
+            name="Product name",
+            price="10",
+            currency="USD",
         )
     )
+    assert item["custom_attrs"] == {
+        "attr1": "foo",
+        "attr2": 42,
+    }
 
 
 class RecordingHandler(ScrapyZyteAPIDownloadHandler):
@@ -1184,9 +1267,9 @@ def parse(self, response: DummyResponse, product: Product):
     assert auto_field_stats == {
         "scrapy-zyte-api/auto_fields/tests.test_providers.test_auto_field_stats_partial_override.<locals>.MyProductPage": (
             "additionalProperties aggregateRating availability breadcrumbs "
-            "canonicalUrl color currency currencyRaw customAttributes description "
-            "descriptionHtml features gtin images mainImage metadata mpn price "
-            "productId regularPrice size sku style url variants"
+            "canonicalUrl color currency currencyRaw description descriptionHtml "
+            "features gtin images mainImage metadata mpn price productId "
+            "regularPrice size sku style url variants"
         ),
     }
 
@@ -1240,10 +1323,6 @@ def currency(self):
         def currencyRaw(self):
             return self.product.currencyRaw
 
-        @field
-        def customAttributes(self):
-            return self.product.customAttributes
-
         @field
         def description(self):
             return self.product.description
@@ -1428,9 +1507,9 @@ def parse(self, response: DummyResponse, page: MyProductPage):
     assert auto_field_stats == {
         "scrapy-zyte-api/auto_fields/tests.test_providers.test_auto_field_stats_item_page_override.<locals>.MyProductPage": (
             "additionalProperties aggregateRating availability breadcrumbs "
-            "canonicalUrl color currency currencyRaw customAttributes description "
-            "descriptionHtml features gtin images mainImage metadata mpn price "
-            "productId regularPrice size sku style url variants"
+            "canonicalUrl color currency currencyRaw description descriptionHtml "
+            "features gtin images mainImage metadata mpn price productId "
+            "regularPrice size sku style url variants"
         ),
     }
 
@@ -1494,9 +1573,9 @@ def parse(self, response: DummyResponse, page: AltProductPage):
     assert auto_field_stats == {
         "scrapy-zyte-api/auto_fields/tests.test_providers.test_auto_field_stats_alt_page_override.<locals>.MyProductPage": (
             "additionalProperties aggregateRating availability breadcrumbs "
-            "canonicalUrl color currency currencyRaw customAttributes description "
-            "descriptionHtml features gtin images mainImage metadata mpn price "
-            "productId regularPrice size sku style url variants"
+            "canonicalUrl color currency currencyRaw description descriptionHtml "
+            "features gtin images mainImage metadata mpn price productId "
+            "regularPrice size sku style url variants"
         ),
     }
 

tox.ini

@@ -90,7 +90,7 @@ deps =
     andi==0.6.0
     scrapy-poet==0.22.3
     web-poet==0.17.0
-    zyte-common-items @ git+https://github.com/zytedata/zyte-common-items.git@custom-attrs
+    zyte-common-items @ git+https://github.com/zytedata/zyte-common-items.git@custom-attrs-dep
 
 [testenv:pinned-extra]
 basepython=python3.8