Merge pull request #106 from Gallaecio/trust-env

Support driving Zyte API requests through a proxy

CHANGES.rst

@@ -18,6 +18,9 @@ Changes
 
 * Improved result caching in the scrapy-poet provider.
 
+* Added a new setting, ``ZYTE_API_USE_ENV_PROXY``, which can be set to ``True``
+  to access Zyte API using a proxy configured in the local environment.
+
 * Fixed getting the Scrapy Cloud job ID.
 
 * Improved the documentation.

README.rst

@@ -913,3 +913,11 @@ We are planning to solve these problems in the future releases of
 ``scrapy-poet`` and ``scrapy-zyte-api``.
 
 .. _scrapy-poet provider: https://scrapy-poet.readthedocs.io/en/stable/providers.html
+
+
+Running behind a proxy
+======================
+
+If you require a proxy to access Zyte API (e.g. a corporate proxy), configure
+the ``HTTP_PROXY`` and ``HTTPS_PROXY`` environment variables accordingly, and
+set the ``ZYTE_API_USE_ENV_PROXY`` setting to ``True``.

scrapy_zyte_api/handler.py

@@ -84,7 +84,10 @@ def __init__(
         self._param_parser = _ParamParser(crawler)
         self._retry_policy = _load_retry_policy(settings)
         self._stats = crawler.stats
-        self._session = create_session(connection_pool_size=self._client.n_conn)
+        self._session = create_session(
+            connection_pool_size=self._client.n_conn,
+            trust_env=settings.getbool("ZYTE_API_USE_ENV_PROXY"),
+        )
         self._must_log_request = settings.getbool("ZYTE_API_LOG_REQUESTS", False)
         self._truncate_limit = settings.getint("ZYTE_API_LOG_REQUESTS_TRUNCATE", 64)
         if self._truncate_limit < 0:

tests/test_handler.py

@@ -430,3 +430,21 @@ def test_log_request_truncate_negative(enabled):
             settings=None,
             crawler=crawler,
         )
+
+
+@pytest.mark.parametrize("enabled", [True, False, None])
+def test_trust_env(enabled):
+    settings: Dict[str, Any] = {
+        **SETTINGS,
+    }
+    if enabled is not None:
+        settings["ZYTE_API_USE_ENV_PROXY"] = enabled
+    else:
+        enabled = False
+    crawler = get_crawler(settings_dict=settings)
+    handler = create_instance(
+        ScrapyZyteAPIDownloadHandler,
+        settings=None,
+        crawler=crawler,
+    )
+    assert handler._session._trust_env == enabled