Skip to content

Commit

Permalink
Check the setting values before overriding.
Browse files Browse the repository at this point in the history
  • Loading branch information
wRAR committed Jul 3, 2023
1 parent b0095f5 commit ce69e94
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 1 deletion.
26 changes: 26 additions & 0 deletions scrapy_zyte_api/addon.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,36 @@
from typing import Optional

from scrapy.settings import BaseSettings

from scrapy_zyte_api import ScrapyZyteAPIDownloaderMiddleware


class ScrapyZyteAPIAddon:
@staticmethod
def _check_settings(settings: BaseSettings) -> Optional[str]:
if (
settings.getwithbase("DOWNLOAD_HANDLERS")["http"]
!= "scrapy.core.downloader.handlers.http.HTTPDownloadHandler"
):
return "'http' value in the 'DOWNLOAD_HANDLERS'"
if (
settings.getwithbase("DOWNLOAD_HANDLERS")["https"]
!= "scrapy.core.downloader.handlers.http.HTTPDownloadHandler"
):
return "'https' value in the 'DOWNLOAD_HANDLERS'"
if (
settings.get("REQUEST_FINGERPRINTER_CLASS")
!= "scrapy.utils.request.RequestFingerprinter"
):
return "'REQUEST_FINGERPRINTER_CLASS'"
return None

def update_settings(self, settings: BaseSettings) -> None:
nondefault_msg = self._check_settings(settings)
if nondefault_msg:
raise ValueError(
f"The {nondefault_msg} setting is set to a custom value, refusing to override it"
)
settings["DOWNLOAD_HANDLERS"][
"http"
] = "scrapy_zyte_api.ScrapyZyteAPIDownloadHandler"
Expand Down
24 changes: 23 additions & 1 deletion tests/test_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,14 @@

from scrapy_zyte_api.handler import ScrapyZyteAPIDownloadHandler

from . import DEFAULT_CLIENT_CONCURRENCY, SETTINGS, UNSET, make_handler, set_env
from . import (
DEFAULT_CLIENT_CONCURRENCY,
SETTINGS,
SETTINGS_ADDON,
UNSET,
make_handler,
set_env,
)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -439,3 +446,18 @@ async def test_addon(mockserver):
request = Request("https://example.com", meta=meta)
await handler.download_request(request, None)
assert handler._stats.get_value("scrapy-zyte-api/success") == 1


@ensureDeferred
async def test_addon_check_settings():
settings: Dict[str, Any]
with pytest.raises(
ValueError, match="The 'http' value in the 'DOWNLOAD_HANDLERS' setting is set"
):
settings = {**SETTINGS_ADDON, "DOWNLOAD_HANDLERS": {"http": "foo"}}
get_crawler(settings_dict=settings)
with pytest.raises(
ValueError, match="The 'REQUEST_FINGERPRINTER_CLASS' setting is set"
):
settings = {**SETTINGS_ADDON, "REQUEST_FINGERPRINTER_CLASS": "foo"}
get_crawler(settings_dict=settings)

0 comments on commit ce69e94

Please sign in to comment.