Merge pull request #100 from scrapy-plugins/more-data-types

Support more data types in the provider
scrapy-plugins · Jul 12, 2023 · bd4bd6b · bd4bd6b
2 parents fd91fbf + 7ac7f12
commit bd4bd6b
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 4 deletions.
diff --git a/README.rst b/README.rst
@@ -30,6 +30,10 @@ Requirements
 * Python 3.7+
 * Scrapy 2.0.1+
 
+scrapy-poet integration requires more recent software:
+
+* Python 3.8+
+* Scrapy 2.6+
 
 Installation
 ============
@@ -859,6 +863,11 @@ The currently supported dependencies are:
 * ``web_poet.BrowserHtml``
 * ``web_poet.BrowserResponse``
 * ``zyte_common_items.Product``
+* ``zyte_common_items.ProductList``
+* ``zyte_common_items.ProductNavigation``
+* ``zyte_common_items.Article``
+* ``zyte_common_items.ArticleList``
+* ``zyte_common_items.ArticleNavigation``
 
 The provider will make a request to Zyte API using the ``ZYTE_API_KEY`` and
 ``ZYTE_API_URL`` settings. It will ignore the transparent mode and parameter

diff --git a/scrapy_zyte_api/providers.py b/scrapy_zyte_api/providers.py
@@ -6,7 +6,14 @@
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy_poet import PageObjectInputProvider
 from web_poet import BrowserHtml, BrowserResponse
-from zyte_common_items import Product
+from zyte_common_items import (
+    Article,
+    ArticleList,
+    ArticleNavigation,
+    Product,
+    ProductList,
+    ProductNavigation,
+)
 
 from scrapy_zyte_api.responses import ZyteAPITextResponse
 
@@ -20,7 +27,16 @@
 class ZyteApiProvider(PageObjectInputProvider):
     name = "zyte_api"
 
-    provided_classes = {BrowserResponse, BrowserHtml, Product}
+    provided_classes = {
+        BrowserResponse,
+        BrowserHtml,
+        Product,
+        ProductList,
+        ProductNavigation,
+        Article,
+        ArticleList,
+        ArticleNavigation,
+    }
 
     def __init__(self, injector):
         super().__init__(injector)
@@ -47,7 +63,14 @@ async def __call__(
             return results
 
         html_requested = BrowserResponse in to_provide or BrowserHtml in to_provide
-        item_keywords = {Product: "product"}
+        item_keywords = {
+            Product: "product",
+            ProductList: "productList",
+            ProductNavigation: "productNavigation",
+            Article: "article",
+            ArticleList: "articleList",
+            ArticleNavigation: "articleNavigation",
+        }
 
         zyte_api_meta = {}
         if html_requested:

diff --git a/setup.py b/setup.py
@@ -19,7 +19,7 @@
         "provider": [
             "scrapy-poet>=0.9.0",
             "web-poet>=0.13.0",
-            "zyte-common-items",
+            "zyte-common-items>=0.7.0",
         ]
     },
     classifiers=[