"
+ assert str(selector) == "True"
+ selector = Selector(text="1")
+ assert repr(selector) == ""
+ assert str(selector) == "1"
+
class ExsltTestCase(unittest.TestCase):
@@ -1214,30 +1231,18 @@ def test_regexp(self) -> None:
# re:test()
self.assertEqual(
sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]').extract(),
- [
- x.extract()
- for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')
- ],
+ [x.extract() for x in sel.xpath('//input[re:test(@name, "[A-Z]+", "i")]')],
)
self.assertEqual(
- [
- x.extract()
- for x in sel.xpath(r'//a[re:test(@href, "\.html$")]/text()')
- ],
+ [x.extract() for x in sel.xpath(r'//a[re:test(@href, "\.html$")]/text()')],
["first link", "second link"],
)
self.assertEqual(
- [
- x.extract()
- for x in sel.xpath('//a[re:test(@href, "first")]/text()')
- ],
+ [x.extract() for x in sel.xpath('//a[re:test(@href, "first")]/text()')],
["first link"],
)
self.assertEqual(
- [
- x.extract()
- for x in sel.xpath('//a[re:test(@href, "second")]/text()')
- ],
+ [x.extract() for x in sel.xpath('//a[re:test(@href, "second")]/text()')],
["second link"],
)
@@ -1267,9 +1272,7 @@ def test_regexp(self) -> None:
r're:replace(//a[re:test(@href, "\.xml$")]/@href,'
r'"(\w+)://(.+)(\.xml)", "","https://\2.html")'
).extract(),
- [
- "https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html"
- ],
+ ["https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html"],
)
def test_set(self) -> None:
@@ -1359,9 +1362,9 @@ def test_text_get(self):
self.assertEqual(txt, "title:\n\nsome text")
def test_text_getall(self):
- sel = self.sscls(
- text=""
- ).getall(text=True)
+ sel = self.sscls(text="").getall(
+ text=True
+ )
self.assertEqual(1, len(sel))
self.assertEqual("option1\noption2", sel[0])
@@ -1399,3 +1402,57 @@ def test_text_xpath_get(self):
html.xpath('//div[@class="product"]/span').getall(text=True),
["Price: 100", "Price: 200"],
)
+
+
+class SelectorBytesInput(Selector):
+ def __init__(
+ self,
+ text: Optional[str] = None,
+ type: Optional[str] = None,
+ body: bytes = b"",
+ encoding: str = "utf8",
+ namespaces: Optional[Mapping[str, str]] = None,
+ root: Optional[Any] = _NOT_SET,
+ base_url: Optional[str] = None,
+ _expr: Optional[str] = None,
+ huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
+ ) -> None:
+ if text:
+ body = bytes(text, encoding=encoding)
+ text = None
+ super().__init__(
+ text=text,
+ type=type,
+ body=body,
+ encoding=encoding,
+ namespaces=namespaces,
+ root=root,
+ base_url=base_url,
+ _expr=_expr,
+ huge_tree=huge_tree,
+ )
+
+
+class SelectorTestCaseBytes(SelectorTestCase):
+ sscls = SelectorBytesInput
+
+ def test_representation_slice(self) -> None:
+ pass
+
+ def test_representation_unicode_query(self) -> None:
+ pass
+
+ def test_weakref_slots(self) -> None:
+ pass
+
+ def test_check_text_argument_type(self) -> None:
+ self.assertRaisesRegex(
+ TypeError,
+ "body argument should be of type",
+ self.sscls,
+ body="",
+ )
+
+
+class ExsltTestCaseBytes(ExsltTestCase):
+ sscls = SelectorBytesInput
diff --git a/tests/test_selector_csstranslator.py b/tests/test_selector_csstranslator.py
index e2934d17..2adc2f5e 100644
--- a/tests/test_selector_csstranslator.py
+++ b/tests/test_selector_csstranslator.py
@@ -1,17 +1,18 @@
"""
Selector tests for cssselect backend
"""
+
import unittest
+from typing import Any, Callable, List, Protocol, Tuple, Type, Union
import cssselect
import pytest
-from packaging.version import Version
-
-from parsel.csstranslator import GenericTranslator, HTMLTranslator
-from parsel import Selector
from cssselect.parser import SelectorSyntaxError
from cssselect.xpath import ExpressionError
+from packaging.version import Version
+from parsel import Selector
+from parsel.csstranslator import GenericTranslator, HTMLTranslator, TranslatorProtocol
HTMLBODY = """
@@ -49,12 +50,32 @@
"""
+class TranslatorTestProtocol(Protocol):
+ tr_cls: Type[TranslatorProtocol]
+ tr: TranslatorProtocol
+
+ def c2x(self, css: str, prefix: str = ...) -> str:
+ pass
+
+ def assertEqual(self, first: Any, second: Any, msg: Any = ...) -> None:
+ pass
+
+ def assertRaises(
+ self,
+ expected_exception: Union[Type[BaseException], Tuple[Type[BaseException], ...]],
+ callable: Callable[..., object],
+ *args: Any,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
+
class TranslatorTestMixin:
- def setUp(self):
+ def setUp(self: TranslatorTestProtocol) -> None:
self.tr = self.tr_cls()
self.c2x = self.tr.css_to_xpath
- def test_attr_function(self):
+ def test_attr_function(self: TranslatorTestProtocol) -> None:
cases = [
("::attr(name)", "descendant-or-self::*/@name"),
("a::attr(href)", "descendant-or-self::a/@href"),
@@ -67,7 +88,7 @@ def test_attr_function(self):
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
- def test_attr_function_exception(self):
+ def test_attr_function_exception(self: TranslatorTestProtocol) -> None:
cases = [
("::attr(12)", ExpressionError),
("::attr(34test)", ExpressionError),
@@ -76,7 +97,7 @@ def test_attr_function_exception(self):
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_text_pseudo_element(self):
+ def test_text_pseudo_element(self: TranslatorTestProtocol) -> None:
cases = [
("::text", "descendant-or-self::text()"),
("p::text", "descendant-or-self::p/text()"),
@@ -105,7 +126,7 @@ def test_text_pseudo_element(self):
for css, xpath in cases:
self.assertEqual(self.c2x(css), xpath, css)
- def test_pseudo_function_exception(self):
+ def test_pseudo_function_exception(self: TranslatorTestProtocol) -> None:
cases = [
("::attribute(12)", ExpressionError),
("::text()", ExpressionError),
@@ -114,14 +135,14 @@ def test_pseudo_function_exception(self):
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_unknown_pseudo_element(self):
+ def test_unknown_pseudo_element(self: TranslatorTestProtocol) -> None:
cases = [
("::text-node", ExpressionError),
]
for css, exc in cases:
self.assertRaises(exc, self.c2x, css)
- def test_unknown_pseudo_class(self):
+ def test_unknown_pseudo_class(self: TranslatorTestProtocol) -> None:
cases = [
(":text", ExpressionError),
(":attribute(name)", ExpressionError),
@@ -139,7 +160,7 @@ class GenericTranslatorTest(TranslatorTestMixin, unittest.TestCase):
class UtilCss2XPathTest(unittest.TestCase):
- def test_css2xpath(self):
+ def test_css2xpath(self) -> None:
from parsel import css2xpath
expected_xpath = (
@@ -153,15 +174,13 @@ class CSSSelectorTest(unittest.TestCase):
sscls = Selector
- def setUp(self):
+ def setUp(self) -> None:
self.sel = self.sscls(text=HTMLBODY)
- def x(self, *a, **kw):
- return [
- v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()
- ]
+ def x(self, *a: Any, **kw: Any) -> List[str]:
+ return [v.strip() for v in self.sel.css(*a, **kw).extract() if v.strip()]
- def test_selector_simple(self):
+ def test_selector_simple(self) -> None:
for x in self.sel.css("input"):
self.assertTrue(isinstance(x, self.sel.__class__), x)
self.assertEqual(
@@ -169,7 +188,7 @@ def test_selector_simple(self):
[x.extract() for x in self.sel.css("input")],
)
- def test_text_pseudo_element(self):
+ def test_text_pseudo_element(self) -> None:
self.assertEqual(self.x("#p-b2"), ['guy'])
self.assertEqual(self.x("#p-b2::text"), ["guy"])
self.assertEqual(self.x("#p-b2 ::text"), ["guy"])
@@ -179,11 +198,9 @@ def test_text_pseudo_element(self):
["lorem ipsum text", "hi", "there", "guy"],
)
self.assertEqual(self.x("p::text"), ["lorem ipsum text"])
- self.assertEqual(
- self.x("p ::text"), ["lorem ipsum text", "hi", "there", "guy"]
- )
+ self.assertEqual(self.x("p ::text"), ["lorem ipsum text", "hi", "there", "guy"])
- def test_attribute_function(self):
+ def test_attribute_function(self) -> None:
self.assertEqual(self.x("#p-b2::attr(id)"), ["p-b2"])
self.assertEqual(self.x(".cool-footer::attr(class)"), ["cool-footer"])
self.assertEqual(
@@ -193,10 +210,8 @@ def test_attribute_function(self):
self.x('map[name="dummymap"] ::attr(shape)'), ["circle", "default"]
)
- def test_nested_selector(self):
- self.assertEqual(
- self.sel.css("p").css("b::text").extract(), ["hi", "guy"]
- )
+ def test_nested_selector(self) -> None:
+ self.assertEqual(self.sel.css("p").css("b::text").extract(), ["hi", "guy"])
self.assertEqual(
self.sel.css("div").css("area:last-child").extract(),
[''],
@@ -206,5 +221,10 @@ def test_nested_selector(self):
Version(cssselect.__version__) < Version("1.2.0"),
reason="Support added in cssselect 1.2.0",
)
- def test_pseudoclass_has(self):
+ def test_pseudoclass_has(self) -> None:
self.assertEqual(self.x("p:has(b)::text"), ["lorem ipsum text"])
+
+
+class CSSSelectorTestBytes(CSSSelectorTest):
+ def setUp(self) -> None:
+ self.sel = self.sscls(body=bytes(HTMLBODY, encoding="utf8"))
diff --git a/tests/test_selector_jmespath.py b/tests/test_selector_jmespath.py
new file mode 100644
index 00000000..5afbd6d0
--- /dev/null
+++ b/tests/test_selector_jmespath.py
@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+
+from parsel import Selector
+from parsel.selector import _NOT_SET
+
+
+class JMESPathTestCase(unittest.TestCase):
+ def test_json_has_html(self) -> None:
+ """Sometimes the information is returned in a json wrapper"""
+ data = """
+ {
+ "content": [
+ {
+ "name": "A",
+ "value": "a"
+ },
+ {
+ "name": {
+ "age": 18
+ },
+ "value": "b"
+ },
+ {
+ "name": "C",
+ "value": "c"
+ },
+ {
+ "name": "D",
+ "value": "d
"
+ }
+ ],
+ "html": ""
+ }
+ """
+ sel = Selector(text=data)
+ self.assertEqual(
+ sel.jmespath("html").get(),
+ "",
+ )
+ self.assertEqual(
+ sel.jmespath("html").xpath("//div/a/text()").getall(),
+ ["a", "b", "d"],
+ )
+ self.assertEqual(sel.jmespath("html").css("div > b").getall(), ["f"])
+ self.assertEqual(sel.jmespath("content").jmespath("name.age").get(), 18)
+
+ def test_html_has_json(self) -> None:
+ html_text = """
+
+
Information
+
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+
+
+ """
+ sel = Selector(text=html_text)
+ self.assertEqual(
+ sel.xpath("//div/content/text()").jmespath("user[*].name").getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("user[*].name").getall(),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(sel.xpath("//div/content").jmespath("total").get(), 4)
+
+ def test_jmestpath_with_re(self) -> None:
+ html_text = """
+
+
Information
+
+ {
+ "user": [
+ {
+ "name": "A",
+ "age": 18
+ },
+ {
+ "name": "B",
+ "age": 32
+ },
+ {
+ "name": "C",
+ "age": 22
+ },
+ {
+ "name": "D",
+ "age": 25
+ }
+ ],
+ "total": 4,
+ "status": "ok"
+ }
+
+
+ """
+ sel = Selector(text=html_text)
+ self.assertEqual(
+ sel.xpath("//div/content/text()").jmespath("user[*].name").re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("user[*].name").re(r"(\w+)"),
+ ["A", "B", "C", "D"],
+ )
+
+ with self.assertRaises(TypeError):
+ sel.xpath("//div/content").jmespath("user[*].age").re(r"(\d+)")
+
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("unavailable").re(r"(\d+)"), []
+ )
+
+ self.assertEqual(
+ sel.xpath("//div/content").jmespath("unavailable").re_first(r"(\d+)"),
+ None,
+ )
+
+ self.assertEqual(
+ sel.xpath("//div/content")
+ .jmespath("user[*].age.to_string(@)")
+ .re(r"(\d+)"),
+ ["18", "32", "22", "25"],
+ )
+
+ def test_json_types(self) -> None:
+ for text, root in (
+ ("{}", {}),
+ ('{"a": "b"}', {"a": "b"}),
+ ("[]", []),
+ ('["a"]', ["a"]),
+ ('""', ""),
+ ("0", 0),
+ ("1", 1),
+ ("true", True),
+ ("false", False),
+ ("null", None),
+ ):
+ selector = Selector(text=text, root=_NOT_SET)
+ self.assertEqual(selector.type, "json")
+ self.assertEqual(selector._text, text) # pylint: disable=protected-access
+ self.assertEqual(selector.root, root)
+
+ selector = Selector(text=None, root=root)
+ self.assertEqual(selector.type, "json")
+ self.assertEqual(selector._text, None) # pylint: disable=protected-access
+ self.assertEqual(selector.root, root)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index e2bca559..ee3e1121 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,7 +1,9 @@
-from parsel.utils import shorten, extract_regex
+from typing import List, Pattern, Type, Union
from pytest import mark, raises
+from parsel.utils import extract_regex, shorten
+
@mark.parametrize(
"width,expected",
@@ -17,7 +19,7 @@
(7, "foobar"),
),
)
-def test_shorten(width, expected):
+def test_shorten(width: int, expected: Union[str, Type[Exception]]) -> None:
if isinstance(expected, str):
assert shorten("foobar", width) == expected
else:
@@ -66,5 +68,10 @@ def test_shorten(width, expected):
],
),
)
-def test_extract_regex(regex, text, replace_entities, expected):
+def test_extract_regex(
+ regex: Union[str, Pattern[str]],
+ text: str,
+ replace_entities: bool,
+ expected: List[str],
+) -> None:
assert extract_regex(regex, text, replace_entities) == expected
diff --git a/tests/test_xml_attacks.py b/tests/test_xml_attacks.py
index 45b0243a..e7b5a486 100644
--- a/tests/test_xml_attacks.py
+++ b/tests/test_xml_attacks.py
@@ -7,11 +7,10 @@
from parsel import Selector
-
MiB_1 = 1024**2
-def _load(attack):
+def _load(attack: str) -> str:
folder_path = path.dirname(__file__)
file_path = path.join(folder_path, "xml_attacks", f"{attack}.xml")
with open(file_path, "rb") as attack_file:
@@ -21,7 +20,7 @@ def _load(attack):
# List of known attacks:
# https://github.com/tiran/defusedxml#python-xml-libraries
class XMLAttackTestCase(TestCase):
- def test_billion_laughs(self):
+ def test_billion_laughs(self) -> None:
process = Process()
memory_usage_before = process.memory_info().rss
selector = Selector(text=_load("billion_laughs"))
diff --git a/tests/test_xpathfuncs.py b/tests/test_xpathfuncs.py
index 744472a9..7739982d 100644
--- a/tests/test_xpathfuncs.py
+++ b/tests/test_xpathfuncs.py
@@ -1,10 +1,12 @@
+import unittest
+from typing import Any
+
from parsel import Selector
from parsel.xpathfuncs import set_xpathfunc
-import unittest
class XPathFuncsTestCase(unittest.TestCase):
- def test_has_class_simple(self):
+ def test_has_class_simple(self) -> None:
body = """
First
Second
@@ -21,21 +23,15 @@ def test_has_class_simple(self):
["Third"],
)
self.assertEqual(
- [
- x.extract()
- for x in sel.xpath('//p[has-class("foo","bar")]/text()')
- ],
+ [x.extract() for x in sel.xpath('//p[has-class("foo","bar")]/text()')],
[],
)
self.assertEqual(
- [
- x.extract()
- for x in sel.xpath('//p[has-class("foo","bar-baz")]/text()')
- ],
+ [x.extract() for x in sel.xpath('//p[has-class("foo","bar-baz")]/text()')],
["First"],
)
- def test_has_class_error_no_args(self):
+ def test_has_class_error_no_args(self) -> None:
body = """
First
"""
@@ -47,7 +43,7 @@ def test_has_class_error_no_args(self):
"has-class()",
)
- def test_has_class_error_invalid_arg_type(self):
+ def test_has_class_error_invalid_arg_type(self) -> None:
body = """
First
"""
@@ -59,7 +55,7 @@ def test_has_class_error_invalid_arg_type(self):
"has-class(.)",
)
- def test_has_class_error_invalid_unicode(self):
+ def test_has_class_error_invalid_unicode(self) -> None:
body = """
First
"""
@@ -71,7 +67,7 @@ def test_has_class_error_invalid_unicode(self):
'has-class("héllö")'.encode(),
)
- def test_has_class_unicode(self):
+ def test_has_class_unicode(self) -> None:
body = """
First
"""
@@ -81,7 +77,7 @@ def test_has_class_unicode(self):
["First"],
)
- def test_has_class_uppercase(self):
+ def test_has_class_uppercase(self) -> None:
body = """
First
"""
@@ -91,7 +87,7 @@ def test_has_class_uppercase(self):
["First"],
)
- def test_has_class_newline(self):
+ def test_has_class_newline(self) -> None:
body = """
First
@@ -102,7 +98,7 @@ def test_has_class_newline(self):
["First"],
)
- def test_has_class_tab(self):
+ def test_has_class_tab(self) -> None:
body = """
First
"""
@@ -112,11 +108,11 @@ def test_has_class_tab(self):
["First"],
)
- def test_set_xpathfunc(self):
- def myfunc(ctx):
- myfunc.call_count += 1
+ def test_set_xpathfunc(self) -> None:
+ def myfunc(ctx: Any) -> None:
+ myfunc.call_count += 1 # type: ignore[attr-defined]
- myfunc.call_count = 0
+ myfunc.call_count = 0 # type: ignore[attr-defined]
body = """
First
@@ -131,7 +127,7 @@ def myfunc(ctx):
set_xpathfunc("myfunc", myfunc)
sel.xpath("myfunc()")
- self.assertEqual(myfunc.call_count, 1)
+ self.assertEqual(myfunc.call_count, 1) # type: ignore[attr-defined]
set_xpathfunc("myfunc", None)
self.assertRaisesRegex(
diff --git a/tests/typing/selector.py b/tests/typing/selector.py
index 72310634..b6f14345 100644
--- a/tests/typing/selector.py
+++ b/tests/typing/selector.py
@@ -1,6 +1,7 @@
# Basic usage of the Selector, strongly typed to test the typing of parsel's API.
import re
from typing import List
+
from parsel import Selector
diff --git a/tox.ini b/tox.ini
index e5e5f47e..893b8de7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
[tox]
-envlist = security,flake8,typing,pylint,black,docs,py37,py38,py39,py310,pypy3.9
+envlist = typing,pylint,docs,twinecheck,pre-commit,py38,py39,py310,py311,py312,pypy3.9,pypy3.10
[testenv]
usedevelop = True
@@ -7,42 +7,25 @@ deps =
-r{toxinidir}/tests/requirements.txt
commands = py.test --cov=parsel --cov-report=xml {posargs:docs parsel tests}
-[testenv:security]
-deps =
- bandit
-commands =
- bandit -r -c .bandit.yml {posargs:parsel}
-
-[testenv:flake8]
-deps =
- {[testenv]deps}
- flake8==5.0.4
-commands =
- flake8 {posargs: parsel tests setup.py}
-
[testenv:typing]
deps =
{[testenv]deps}
- types-lxml==2022.4.10
- types-psutil==5.9.5.4
- types-setuptools==65.5.0.1
- mypy==0.982
+ types-jmespath==1.0.2.20240106
+ types-lxml==2024.2.9
+ types-psutil==5.9.5.20240311
+ types-setuptools==69.1.0.20240310
+ py==1.11.0
+ mypy==1.9.0
commands =
- mypy {posargs:parsel tests} --warn-unused-ignores --show-error-codes
+ mypy {posargs:parsel tests} --strict
[testenv:pylint]
deps =
{[testenv]deps}
- pylint==2.15.4
+ pylint==3.1.0
commands =
pylint docs parsel tests setup.py
-[testenv:black]
-deps =
- black==22.10.0
-commands =
- black --line-length=79 --check {posargs:parsel tests setup.py}
-
[docs]
changedir = docs
deps = -rdocs/requirements.txt
@@ -55,4 +38,18 @@ deps = {[docs]deps}
commands =
sphinx-build -W -b html . {envtmpdir}/html
sphinx-build -b latex . {envtmpdir}/latex
- sphinx-build -W -b epub . {envtmpdir}/epub
+ sphinx-build -b epub . {envtmpdir}/epub
+
+[testenv:twinecheck]
+basepython = python3
+deps =
+ twine==5.0.0
+ build==1.1.1
+commands =
+ python -m build --sdist
+ twine check dist/*
+
+[testenv:pre-commit]
+deps = pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
+skip_install = true