From 5ad4d7944d1dc2379e59afd5590d10be33faf5e2 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Jul 2023 17:44:37 +0500 Subject: [PATCH 1/5] ZyteApiProvider: support more data types --- scrapy_zyte_api/providers.py | 25 ++++++++++++++++++++++--- setup.py | 2 +- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scrapy_zyte_api/providers.py b/scrapy_zyte_api/providers.py index 26359a85..e76caa9e 100644 --- a/scrapy_zyte_api/providers.py +++ b/scrapy_zyte_api/providers.py @@ -6,7 +6,10 @@ from scrapy.utils.defer import maybe_deferred_to_future from scrapy_poet import PageObjectInputProvider from web_poet import BrowserHtml, BrowserResponse -from zyte_common_items import Product +from zyte_common_items import ( + Product, ProductList, ProductNavigation, + Article, ArticleList, ArticleNavigation +) from scrapy_zyte_api.responses import ZyteAPITextResponse @@ -20,7 +23,16 @@ class ZyteApiProvider(PageObjectInputProvider): name = "zyte_api" - provided_classes = {BrowserResponse, BrowserHtml, Product} + provided_classes = { + BrowserResponse, + BrowserHtml, + Product, + ProductList, + ProductNavigation, + Article, + ArticleList, + ArticleNavigation + } def __init__(self, injector): super().__init__(injector) @@ -47,7 +59,14 @@ async def __call__( return results html_requested = BrowserResponse in to_provide or BrowserHtml in to_provide - item_keywords = {Product: "product"} + item_keywords = { + Product: "product", + ProductList: "productList", + ProductNavigation: "productNavigation", + Article: "article", + ArticleList: "articleList", + ArticleNavigation: "articleNavigation", + } zyte_api_meta = {} if html_requested: diff --git a/setup.py b/setup.py index 6c340a2d..a4109ae5 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ "provider": [ "scrapy-poet>=0.9.0", "web-poet>=0.13.0", - "zyte-common-items", + "zyte-common-items>=0.7.0", ] }, classifiers=[ From 914d5998cb75fe3d9cfe2a53e52f8f5548961fc7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Jul 2023 17:47:01 +0500 Subject: [PATCH 2/5] pre-commit fixes --- scrapy_zyte_api/providers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy_zyte_api/providers.py b/scrapy_zyte_api/providers.py index e76caa9e..384ceada 100644 --- a/scrapy_zyte_api/providers.py +++ b/scrapy_zyte_api/providers.py @@ -7,8 +7,12 @@ from scrapy_poet import PageObjectInputProvider from web_poet import BrowserHtml, BrowserResponse from zyte_common_items import ( - Product, ProductList, ProductNavigation, - Article, ArticleList, ArticleNavigation + Article, + ArticleList, + ArticleNavigation, + Product, + ProductList, + ProductNavigation, ) from scrapy_zyte_api.responses import ZyteAPITextResponse @@ -31,7 +35,7 @@ class ZyteApiProvider(PageObjectInputProvider): ProductNavigation, Article, ArticleList, - ArticleNavigation + ArticleNavigation, } def __init__(self, injector): From 3c81296cdfb2d451b0199157d8696f54677ed3b6 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Jul 2023 17:51:52 +0500 Subject: [PATCH 3/5] document requirements for scrapy-poet integration --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index a095b399..ecaed389 100644 --- a/README.rst +++ b/README.rst @@ -30,6 +30,10 @@ Requirements * Python 3.7+ * Scrapy 2.0.1+ +Scrapy-poet integration requires more recent software: + +* Python 3.8+ +* Scrapy 2.6+ Installation ============ From 44c3a5f423e124bdcbc4bc85d3e391b16261a0f6 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Jul 2023 19:03:27 +0500 Subject: [PATCH 4/5] document new dependencies which scrapy-zyte-api can provide --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index ecaed389..5e65408b 100644 --- a/README.rst +++ b/README.rst @@ -863,6 +863,11 @@ The currently supported dependencies are: * ``web_poet.BrowserHtml`` * ``web_poet.BrowserResponse`` * ``zyte_common_items.Product`` +* ``zyte_common_items.ProductList`` +* ``zyte_common_items.ProductNavigation`` +* ``zyte_common_items.Article`` +* ``zyte_common_items.ArticleList`` +* ``zyte_common_items.ArticleNavigation`` The provider will make a request to Zyte API using the ``ZYTE_API_KEY`` and ``ZYTE_API_URL`` settings. It will ignore the transparent mode and parameter From 7ac7f1298f0f3bb1f3e5f46b519cbf71bd3d4fb5 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 12 Jul 2023 21:05:38 +0500 Subject: [PATCH 5/5] Update README.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5e65408b..b4449ed0 100644 --- a/README.rst +++ b/README.rst @@ -30,7 +30,7 @@ Requirements * Python 3.7+ * Scrapy 2.0.1+ -Scrapy-poet integration requires more recent software: +scrapy-poet integration requires more recent software: * Python 3.8+ * Scrapy 2.6+