From 6e0ee6463e73eb5e893b6b1aa3d046914bd7df43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 30 Sep 2024 12:52:01 -0700 Subject: [PATCH] Implement string replacement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- src/tox/config/loader/ini/__init__.py | 6 +- src/tox/config/loader/ini/replace.py | 382 +++--------------- src/tox/config/loader/replacer.py | 297 ++++++++++++++ src/tox/config/loader/stringify.py | 4 +- src/tox/config/loader/toml/__init__.py | 21 +- src/tox/config/loader/toml/_replace.py | 177 ++++++-- src/tox/config/loader/toml/_validate.py | 8 - src/tox/config/of_type.py | 16 +- src/tox/config/set_env.py | 7 +- src/tox/config/sets.py | 5 +- src/tox/config/source/toml_pyproject.py | 27 +- .../loader/{ini/replace => }/conftest.py | 0 .../ini/replace/test_replace_tox_env.py | 2 +- .../loader/{ini/replace => }/test_replace.py | 2 +- tests/config/loader/test_toml_loader.py | 24 +- tests/config/source/test_toml_pyproject.py | 174 ++++++++ tests/config/source/test_toml_tox.py | 52 ++- tox.toml | 18 +- 18 files changed, 797 insertions(+), 425 deletions(-) create mode 100644 src/tox/config/loader/replacer.py rename tests/config/loader/{ini/replace => }/conftest.py (100%) rename tests/config/loader/{ini/replace => }/test_replace.py (97%) diff --git a/src/tox/config/loader/ini/__init__.py b/src/tox/config/loader/ini/__init__.py index 5f0e5eda7..9d97a3312 100644 --- a/src/tox/config/loader/ini/__init__.py +++ b/src/tox/config/loader/ini/__init__.py @@ -6,7 +6,8 @@ from tox.config.loader.api import ConfigLoadArgs, Loader, Override from tox.config.loader.ini.factor import filter_for_env -from tox.config.loader.ini.replace import replace +from tox.config.loader.ini.replace import ReplaceReferenceIni +from tox.config.loader.replacer import replace from tox.config.loader.str_convert import StrConvert from tox.config.set_env import SetEnv from tox.report import HandledError @@ -71,8 +72,9 @@ def replacer(raw_: str, args_: ConfigLoadArgs) -> str: if conf is None: replaced = raw_ # no replacement supported in the core section else: + reference_replacer = ReplaceReferenceIni(conf, self) try: - replaced = replace(conf, self, raw_, args_) # do replacements + replaced = replace(conf, reference_replacer, raw_, args_) # do replacements except Exception as exception: if isinstance(exception, HandledError): raise diff --git a/src/tox/config/loader/ini/replace.py b/src/tox/config/loader/ini/replace.py index 9037f2674..f225cedbc 100644 --- a/src/tox/config/loader/ini/replace.py +++ b/src/tox/config/loader/ini/replace.py @@ -2,222 +2,84 @@ from __future__ import annotations -import logging -import os import re -import sys from configparser import SectionProxy from functools import lru_cache -from typing import TYPE_CHECKING, Any, Iterator, Pattern, Sequence, Union +from typing import TYPE_CHECKING, Iterator, Pattern +from tox.config.loader.replacer import ReplaceReference from tox.config.loader.stringify import stringify -from tox.execute.request import shell_cmd if TYPE_CHECKING: - from pathlib import Path - from tox.config.loader.api import ConfigLoadArgs from tox.config.loader.ini import IniLoader from tox.config.main import Config - from tox.config.set_env import SetEnv from tox.config.sets import ConfigSet -LOGGER = logging.getLogger(__name__) - - -# split alongside :, unless it's preceded by a single capital letter (Windows drive letter in paths) -ARG_DELIMITER = ":" -REPLACE_START = "{" -REPLACE_END = "}" -BACKSLASH_ESCAPE_CHARS = [ARG_DELIMITER, REPLACE_START, REPLACE_END, "[", "]"] -MAX_REPLACE_DEPTH = 100 - - -MatchArg = Sequence[Union[str, "MatchExpression"]] - - -class MatchRecursionError(ValueError): - """Could not stabilize on replacement value.""" - - -class MatchError(Exception): - """Could not find end terminator in MatchExpression.""" - - -def find_replace_expr(value: str) -> MatchArg: - """Find all replaceable tokens within value.""" - return MatchExpression.parse_and_split_to_terminator(value)[0][0] - - -def replace(conf: Config, loader: IniLoader, value: str, args: ConfigLoadArgs, depth: int = 0) -> str: - """Replace all active tokens within value according to the config.""" - if depth > MAX_REPLACE_DEPTH: - msg = f"Could not expand {value} after recursing {depth} frames" - raise MatchRecursionError(msg) - return Replacer(conf, loader, conf_args=args, depth=depth).join(find_replace_expr(value)) - - -class MatchExpression: # noqa: PLW1641 - """An expression that is handled specially by the Replacer.""" - - def __init__(self, expr: Sequence[MatchArg], term_pos: int | None = None) -> None: - self.expr = expr - self.term_pos = term_pos - - def __repr__(self) -> str: - return f"MatchExpression(expr={self.expr!r}, term_pos={self.term_pos!r})" - - def __eq__(self, other: object) -> bool: - if isinstance(other, type(self)): - return self.expr == other.expr - return NotImplemented - - @classmethod - def _next_replace_expression(cls, value: str) -> MatchExpression | None: - """Process a curly brace replacement expression.""" - if value.startswith("[]"): - # `[]` is shorthand for `{posargs}` - return MatchExpression(expr=[["posargs"]], term_pos=1) - if not value.startswith(REPLACE_START): - return None - try: - # recursively handle inner expression - rec_expr, term_pos = cls.parse_and_split_to_terminator( - value[1:], - terminator=REPLACE_END, - split=ARG_DELIMITER, - ) - except MatchError: - # did NOT find the expected terminator character, so treat `{` as if escaped - pass - else: - return MatchExpression(expr=rec_expr, term_pos=term_pos) - return None - - @classmethod - def parse_and_split_to_terminator( - cls, - value: str, - terminator: str = "", - split: str | None = None, - ) -> tuple[Sequence[MatchArg], int]: - """ - Tokenize `value` to up `terminator` character. - - If `split` is given, multiple arguments will be returned. - - Returns list of arguments (list of str or MatchExpression) and final character position examined in value. - - This function recursively calls itself via `_next_replace_expression`. - """ - args = [] - last_arg: list[str | MatchExpression] = [] - pos = 0 - - while pos < len(value): - if len(value) > pos + 1 and value[pos] == "\\": - if value[pos + 1] in BACKSLASH_ESCAPE_CHARS: - # backslash escapes the next character from a special set - last_arg.append(value[pos + 1]) - pos += 2 - continue - if value[pos + 1] == "\\": - # backlash doesn't escape a backslash, but does prevent it from affecting the next char - # a subsequent `shlex` pass will eat the double backslash during command splitting - last_arg.append(value[pos : pos + 2]) - pos += 2 - continue - fragment = value[pos:] - if terminator and fragment.startswith(terminator): - pos += len(terminator) - break - if split and fragment.startswith(split): - # found a new argument - args.append(last_arg) - last_arg = [] - pos += len(split) - continue - expr = cls._next_replace_expression(fragment) - if expr is not None: - pos += (expr.term_pos or 0) + 1 - last_arg.append(expr) - continue - # default case: consume the next character - last_arg.append(value[pos]) - pos += 1 - else: # fell out of the loop - if terminator: - msg = f"{terminator!r} remains unmatched in {value!r}" - raise MatchError(msg) - args.append(last_arg) - return [_flatten_string_fragments(a) for a in args], pos - - -def _flatten_string_fragments(seq_of_str_or_other: Sequence[str | Any]) -> Sequence[str | Any]: - """Join runs of contiguous str values in a sequence; nny non-str items in the sequence are left as-is.""" - result = [] - last_str = [] - for obj in seq_of_str_or_other: - if isinstance(obj, str): - last_str.append(obj) - else: - if last_str: - result.append("".join(last_str)) - last_str = [] - result.append(obj) - if last_str: - result.append("".join(last_str)) - return result - - -class Replacer: - """Recursively expand MatchExpression against the config and loader.""" - - def __init__(self, conf: Config, loader: IniLoader, conf_args: ConfigLoadArgs, depth: int = 0) -> None: +class ReplaceReferenceIni(ReplaceReference): + def __init__(self, conf: Config, loader: IniLoader) -> None: self.conf = conf self.loader = loader - self.conf_args = conf_args - self.depth = depth - def __call__(self, value: MatchArg) -> Sequence[str]: - return [self._replace_match(me) if isinstance(me, MatchExpression) else str(me) for me in value] - - def join(self, value: MatchArg) -> str: - return "".join(self(value)) + def __call__(self, value: str, conf_args: ConfigLoadArgs) -> str | None: # noqa: C901 + # a return value of None indicates could not replace + pattern = _replace_ref(self.loader.section.prefix or self.loader.section.name) + match = pattern.match(value) + if match: + settings = match.groupdict() + + key = settings["key"] + if settings["section"] is None and settings["full_env"]: + settings["section"] = settings["full_env"] + + exception: Exception | None = None + try: + for src in self._config_value_sources(settings["env"], settings["section"], conf_args.env_name): + try: + if isinstance(src, SectionProxy): + return self.loader.process_raw(self.conf, conf_args.env_name, src[key]) + value = src.load(key, conf_args.chain) + except KeyError as exc: # if fails, keep trying maybe another source can satisfy # noqa: PERF203 + exception = exc + else: + as_str, _ = stringify(value) + return as_str.replace("#", r"\#") # escape comment characters as these will be stripped + except Exception as exc: # noqa: BLE001 + exception = exc + if exception is not None: + if isinstance(exception, KeyError): # if the lookup failed replace - else keep + default = settings["default"] + if default is not None: + return default + # we cannot raise here as that would mean users could not write factorials: + # depends = {py39,py38}-{,b} + else: + raise exception + return None - def _replace_match(self, value: MatchExpression) -> str: - # use a copy of conf_args so any changes from this replacement do NOT, affect adjacent substitutions (#2869) - conf_args = self.conf_args.copy() - flattened_args = [self.join(arg) for arg in value.expr] - of_type, *args = flattened_args - if of_type == "/": - replace_value: str | None = os.sep - elif not of_type and args == [""]: - replace_value = os.pathsep - elif of_type == "env": - replace_value = replace_env(self.conf, args, conf_args) - elif of_type == "tty": - replace_value = replace_tty(args) - elif of_type == "posargs": - replace_value = replace_pos_args(self.conf, args, conf_args) - else: - arg_value = ARG_DELIMITER.join(flattened_args) - replace_value = replace_reference(self.conf, self.loader, arg_value, conf_args) - if replace_value is not None: - needs_expansion = any(isinstance(m, MatchExpression) for m in find_replace_expr(replace_value)) - if needs_expansion: - try: - return replace(self.conf, self.loader, replace_value, conf_args, self.depth + 1) - except MatchRecursionError as err: - LOGGER.warning(str(err)) - return replace_value - return replace_value - # else: fall through -- when replacement is not possible, treat `{` as if escaped. - # If we cannot replace, keep what was there, and continue looking for additional replaces - # NOTE: cannot raise because the content may be a factorial expression where we don't - # want to enforce escaping curly braces, e.g. `env_list = {py39,py38}-{,dep}` should work - return f"{REPLACE_START}%s{REPLACE_END}" % ARG_DELIMITER.join(flattened_args) + def _config_value_sources( + self, env: str | None, section: str | None, current_env: str | None + ) -> Iterator[SectionProxy | ConfigSet]: + # if we have an env name specified take only from there + if env is not None and env in self.conf: + yield self.conf.get_env(env) + + if section is None: + # if no section specified perhaps it's an unregistered config: + # 1. try first from core conf + yield self.conf.core + # 2. and then fallback to our own environment + if current_env is not None: + yield self.conf.get_env(current_env) + return + + # if there's a section, special handle the core section + if section == self.loader.core_section.name: + yield self.conf.core # try via registered configs + value = self.loader.get_section(section) # fallback to section + if value is not None: + yield value @lru_cache(maxsize=None) @@ -233,122 +95,6 @@ def _replace_ref(env: str | None) -> Pattern[str]: ) -def replace_reference( # noqa: C901 - conf: Config, - loader: IniLoader, - value: str, - conf_args: ConfigLoadArgs, -) -> str | None: - # a return value of None indicates could not replace - pattern = _replace_ref(loader.section.prefix or loader.section.name) - match = pattern.match(value) - if match: - settings = match.groupdict() - - key = settings["key"] - if settings["section"] is None and settings["full_env"]: - settings["section"] = settings["full_env"] - - exception: Exception | None = None - try: - for src in _config_value_sources(settings["env"], settings["section"], conf_args.env_name, conf, loader): - try: - if isinstance(src, SectionProxy): - return loader.process_raw(conf, conf_args.env_name, src[key]) - value = src.load(key, conf_args.chain) - except KeyError as exc: # if fails, keep trying maybe another source can satisfy # noqa: PERF203 - exception = exc - else: - as_str, _ = stringify(value) - return as_str.replace("#", r"\#") # escape comment characters as these will be stripped - except Exception as exc: # noqa: BLE001 - exception = exc - if exception is not None: - if isinstance(exception, KeyError): # if the lookup failed replace - else keep - default = settings["default"] - if default is not None: - return default - # we cannot raise here as that would mean users could not write factorials: depends = {py39,py38}-{,b} - else: - raise exception - return None - - -def _config_value_sources( - env: str | None, - section: str | None, - current_env: str | None, - conf: Config, - loader: IniLoader, -) -> Iterator[SectionProxy | ConfigSet]: - # if we have an env name specified take only from there - if env is not None and env in conf: - yield conf.get_env(env) - - if section is None: - # if no section specified perhaps it's an unregistered config: - # 1. try first from core conf - yield conf.core - # 2. and then fallback to our own environment - if current_env is not None: - yield conf.get_env(current_env) - return - - # if there's a section, special handle the core section - if section == loader.core_section.name: - yield conf.core # try via registered configs - value = loader.get_section(section) # fallback to section - if value is not None: - yield value - - -def replace_pos_args(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -> str: - to_path: Path | None = None - if conf_args.env_name is not None: # pragma: no branch - env_conf = conf.get_env(conf_args.env_name) - try: - if env_conf["args_are_paths"]: # pragma: no branch - to_path = env_conf["change_dir"] - except KeyError: - pass - pos_args = conf.pos_args(to_path) - # if we use the defaults join back remaining args else take shell cmd - return ARG_DELIMITER.join(args) if pos_args is None else shell_cmd(pos_args) - - -def replace_env(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -> str: - if not args or not args[0]: - msg = "No variable name was supplied in {env} substitution" - raise MatchError(msg) - key = args[0] - new_key = f"env:{key}" - - if conf_args.env_name is not None: # on core no set env support # pragma: no branch - if new_key not in conf_args.chain: # check if set env - conf_args.chain.append(new_key) - env_conf = conf.get_env(conf_args.env_name) - set_env: SetEnv = env_conf["set_env"] - if key in set_env: - return set_env.load(key, conf_args) - elif conf_args.chain[-1] != new_key: # if there's a chain but only self-refers than use os.environ - circular = ", ".join(i[4:] for i in conf_args.chain[conf_args.chain.index(new_key) :]) - msg = f"circular chain between set env {circular}" - raise MatchRecursionError(msg) - - if key in os.environ: - return os.environ[key] - - return "" if len(args) == 1 else ARG_DELIMITER.join(args[1:]) - - -def replace_tty(args: list[str]) -> str: - return (args[0] if len(args) > 0 else "") if sys.stdout.isatty() else args[1] if len(args) > 1 else "" - - -__all__ = ( - "MatchArg", - "MatchError", - "MatchExpression", - "find_replace_expr", - "replace", -) +__all__ = [ + "ReplaceReferenceIni", +] diff --git a/src/tox/config/loader/replacer.py b/src/tox/config/loader/replacer.py new file mode 100644 index 000000000..457a2b4fd --- /dev/null +++ b/src/tox/config/loader/replacer.py @@ -0,0 +1,297 @@ +"""Apply value substitution (replacement) on tox strings.""" + +from __future__ import annotations + +import logging +import os +import sys +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Final, Sequence, Union + +from tox.config.of_type import CircularChainError +from tox.execute.request import shell_cmd + +if TYPE_CHECKING: + from pathlib import Path + + from tox.config.loader.api import ConfigLoadArgs + from tox.config.main import Config + from tox.config.set_env import SetEnv + + +LOGGER = logging.getLogger(__name__) + +# split alongside :, unless it is preceded by a single capital letter (Windows drive letters in paths) +ARG_DELIMITER: Final[str] = ":" +REPLACE_START: Final[str] = "{" +REPLACE_END: Final[str] = "}" +BACKSLASH_ESCAPE_CHARS: Final[tuple[str, ...]] = (ARG_DELIMITER, REPLACE_START, REPLACE_END, "[", "]") +MAX_REPLACE_DEPTH: Final[int] = 100 + + +class MatchRecursionError(ValueError): + """Could not stabilize on replacement value.""" + + @staticmethod + def check(depth: int, value: Any) -> None: + if depth > MAX_REPLACE_DEPTH: + msg = f"Could not expand {value} after recursing {depth} frames" + raise MatchRecursionError(msg) + + +class MatchError(Exception): + """Couldn't find end terminator in MatchExpression.""" + + +class ReplaceReference(ABC): + @abstractmethod + def __call__(self, value: str, conf_args: ConfigLoadArgs) -> str | None: + """ + Perform a reference replacement. + + :param value: the raw value + :param conf_args: the configuration loads argument object + :return: the replaced value, None if it can't do it. + """ + raise NotImplementedError + + +MatchArg = Sequence[Union[str, "MatchExpression"]] + + +def find_replace_expr(value: str) -> MatchArg: + """Find all replaceable tokens within value.""" + return MatchExpression.parse_and_split_to_terminator(value)[0][0] + + +def replace(conf: Config, reference: ReplaceReference, value: str, args: ConfigLoadArgs, depth: int = 0) -> str: + """Replace all active tokens within value according to the config.""" + MatchRecursionError.check(depth, value) + return Replacer(conf, reference, conf_args=args, depth=depth).join(find_replace_expr(value)) + + +class MatchExpression: # noqa: PLW1641 + """An expression that is handled specially by the Replacer.""" + + def __init__(self, expr: Sequence[MatchArg], term_pos: int | None = None) -> None: + self.expr = expr + self.term_pos = term_pos + + def __repr__(self) -> str: + return f"MatchExpression(expr={self.expr!r}, term_pos={self.term_pos!r})" + + def __eq__(self, other: object) -> bool: + if isinstance(other, type(self)): + return self.expr == other.expr + return NotImplemented + + @classmethod + def _next_replace_expression(cls, value: str) -> MatchExpression | None: + """Process a curly brace replacement expression.""" + if value.startswith("[]"): + # `[]` is shorthand for `{posargs}` + return MatchExpression(expr=[["posargs"]], term_pos=1) + if not value.startswith(REPLACE_START): + return None + try: + # recursively handle inner expression + rec_expr, term_pos = cls.parse_and_split_to_terminator( + value[1:], + terminator=REPLACE_END, + split=ARG_DELIMITER, + ) + except MatchError: + # didn't find the expected terminator character, so treat `{` as if escaped. + pass + else: + return MatchExpression(expr=rec_expr, term_pos=term_pos) + return None + + @classmethod + def parse_and_split_to_terminator( + cls, + value: str, + terminator: str = "", + split: str | None = None, + ) -> tuple[Sequence[MatchArg], int]: + """ + Tokenize `value` to up `terminator` character. + + If `split` is given, multiple arguments will be returned. + + Returns list of arguments (list of str or MatchExpression) and final character position examined in value. + + This function recursively calls itself via `_next_replace_expression`. + """ + args = [] + last_arg: list[str | MatchExpression] = [] + pos = 0 + + while pos < len(value): + if len(value) > pos + 1 and value[pos] == "\\": + if value[pos + 1] in BACKSLASH_ESCAPE_CHARS: + # backslash escapes the next character from a special set + last_arg.append(value[pos + 1]) + pos += 2 + continue + if value[pos + 1] == "\\": + # backlash doesn't escape a backslash, but does prevent it from affecting the next char + # a subsequent `shlex` pass will eat the double backslash during command splitting. + last_arg.append(value[pos : pos + 2]) + pos += 2 + continue + fragment = value[pos:] + if terminator and fragment.startswith(terminator): + pos += len(terminator) + break + if split and fragment.startswith(split): + # found a new argument + args.append(last_arg) + last_arg = [] + pos += len(split) + continue + expr = cls._next_replace_expression(fragment) + if expr is not None: + pos += (expr.term_pos or 0) + 1 + last_arg.append(expr) + continue + # default case: consume the next character + last_arg.append(value[pos]) + pos += 1 + else: # fell out of the loop + if terminator: + msg = f"{terminator!r} remains unmatched in {value!r}" + raise MatchError(msg) + args.append(last_arg) + return [_flatten_string_fragments(a) for a in args], pos + + +def _flatten_string_fragments(seq_of_str_or_other: Sequence[str | Any]) -> Sequence[str | Any]: + """Join runs of contiguous str values in a sequence; nny non-str items in the sequence are left as-is.""" + result = [] + last_str = [] + for obj in seq_of_str_or_other: + if isinstance(obj, str): + last_str.append(obj) + else: + if last_str: + result.append("".join(last_str)) + last_str = [] + result.append(obj) + if last_str: + result.append("".join(last_str)) + return result + + +class Replacer: + """Recursively expand MatchExpression against the config and loader.""" + + def __init__(self, conf: Config, reference: ReplaceReference, conf_args: ConfigLoadArgs, depth: int = 0) -> None: + self.conf = conf + self.reference = reference + self.conf_args = conf_args + self.depth = depth + + def __call__(self, value: MatchArg) -> Sequence[str]: + return [self._replace_match(me) if isinstance(me, MatchExpression) else str(me) for me in value] + + def join(self, value: MatchArg) -> str: + return "".join(self(value)) + + def _replace_match(self, value: MatchExpression) -> str: + # use a copy of conf_args so any changes from this replacement don't, affect adjacent substitutions (#2869) + conf_args = self.conf_args.copy() + flattened_args = [self.join(arg) for arg in value.expr] + of_type, *args = flattened_args + if of_type == "/": + replace_value: str | None = os.sep + elif not of_type and args == [""]: + replace_value = os.pathsep + elif of_type == "env": + replace_value = replace_env(self.conf, args, conf_args) + elif of_type == "tty": + replace_value = replace_tty(args) + elif of_type == "posargs": + replace_value = replace_pos_args(self.conf, args, conf_args) + else: + arg_value = ARG_DELIMITER.join(flattened_args) + replace_value = self.reference(arg_value, conf_args) + if replace_value is not None: + needs_expansion = any(isinstance(m, MatchExpression) for m in find_replace_expr(replace_value)) + if needs_expansion: + try: + return replace(self.conf, self.reference, replace_value, conf_args, self.depth + 1) + except MatchRecursionError as err: + LOGGER.warning(str(err)) + return replace_value + return replace_value + # else: fall through -- when replacement is impossible, treat `{` as if escaped. + # If we can't replace, keep what was there, and continue looking for additional replaces + # NOTE: can't raise because the content may be a factorial expression where we don't + # want to enforce escaping curly braces, for example`env_list = {py39,py38}-{,dep}` should work + return f"{REPLACE_START}%s{REPLACE_END}" % ARG_DELIMITER.join(flattened_args) + + +def replace_pos_args(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -> str: + pos_args = load_posargs(conf, conf_args) + # if we use the defaults, join back remaining args else take shell cmd. + return ARG_DELIMITER.join(args) if pos_args is None else shell_cmd(pos_args) + + +def load_posargs(conf: Config, conf_args: ConfigLoadArgs) -> tuple[str, ...] | None: + to_path: Path | None = None + if conf_args.env_name is not None: # pragma: no branch + env_conf = conf.get_env(conf_args.env_name) + try: + if env_conf["args_are_paths"]: # pragma: no branch + to_path = env_conf["change_dir"] + except KeyError: + pass + return conf.pos_args(to_path) + + +def replace_env(conf: Config | None, args: list[str], conf_args: ConfigLoadArgs) -> str: + if not args or not args[0]: + msg = "No variable name was supplied in {env} substitution" + raise MatchError(msg) + key = args[0] + new_key = f"env:{key}" + + if conf is not None and conf_args.env_name is not None: # on core no set env support # pragma: no branch + if new_key not in conf_args.chain: # check if set env + conf_args.chain.append(new_key) + env_conf = conf.get_env(conf_args.env_name) + try: + set_env: SetEnv = env_conf.load("set_env", chain=conf_args.chain) + except CircularChainError: + if not ( + conf_args.chain[-1].endswith(".set_env") + and any(i.endswith(".set_env") for i in conf_args.chain[:-1]) + ): + raise + else: + if key in set_env: + return set_env.load(key, conf_args) + elif conf_args.chain[-1] != new_key: # if there's a chain but only self-refers than use os.environ + circular = ", ".join(i[4:] for i in conf_args.chain[conf_args.chain.index(new_key) :]) + msg = f"circular chain between set env {circular}" + raise MatchRecursionError(msg) + + if key in os.environ: + return os.environ[key] + + return "" if len(args) == 1 else ARG_DELIMITER.join(args[1:]) + + +def replace_tty(args: list[str]) -> str: + return (args[0] if len(args) > 0 else "") if sys.stdout.isatty() else args[1] if len(args) > 1 else "" + + +__all__ = [ + "MatchExpression", + "MatchRecursionError", + "find_replace_expr", + "load_posargs", + "replace", + "replace_env", +] diff --git a/src/tox/config/loader/stringify.py b/src/tox/config/loader/stringify.py index 2ff026df8..94d7d96ab 100644 --- a/src/tox/config/loader/stringify.py +++ b/src/tox/config/loader/stringify.py @@ -21,8 +21,10 @@ def stringify(value: Any) -> tuple[str, bool]: # noqa: PLR0911 return str(value), False if isinstance(value, Mapping): return "\n".join(f"{stringify(k)[0]}={stringify(v)[0]}" for k, v in value.items()), True - if isinstance(value, (Sequence, Set)): + if isinstance(value, Sequence): return "\n".join(stringify(i)[0] for i in value), True + if isinstance(value, Set): # sort it to make it stable + return "\n".join(sorted(stringify(i)[0] for i in value)), True if isinstance(value, EnvList): return "\n".join(e for e in value.envs), True if isinstance(value, Command): diff --git a/src/tox/config/loader/toml/__init__.py b/src/tox/config/loader/toml/__init__.py index e636bfdf6..177fcd534 100644 --- a/src/tox/config/loader/toml/__init__.py +++ b/src/tox/config/loader/toml/__init__.py @@ -1,13 +1,13 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, Set, TypeVar, cast +from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, TypeVar, cast from tox.config.loader.api import ConfigLoadArgs, Loader, Override from tox.config.types import Command, EnvList from ._api import TomlTypes -from ._replace import unroll_refs_and_apply_substitutions +from ._replace import Unroll from ._validate import validate if TYPE_CHECKING: @@ -27,9 +27,11 @@ def __init__( section: Section, overrides: list[Override], content: Mapping[str, TomlTypes], + root_content: Mapping[str, TomlTypes], unused_exclude: set[str], ) -> None: self.content = content + self._root_content = root_content self._unused_exclude = unused_exclude super().__init__(section, overrides) @@ -39,6 +41,15 @@ def __repr__(self) -> str: def load_raw(self, key: str, conf: Config | None, env_name: str | None) -> TomlTypes: # noqa: ARG002 return self.content[key] + def load_raw_from_root(self, path: str) -> TomlTypes: + current = cast(TomlTypes, self._root_content) + for key in path.split(self.section.SEP): + if isinstance(current, dict) and key in current: + current = current[key] + else: + return None + return current + def build( # noqa: PLR0913 self, key: str, # noqa: ARG002 @@ -48,8 +59,8 @@ def build( # noqa: PLR0913 raw: TomlTypes, args: ConfigLoadArgs, ) -> _T: - raw = unroll_refs_and_apply_substitutions(conf=conf, loader=self, value=raw, args=args) - return self.to(raw, of_type, factory) + exploded = Unroll(conf=conf, loader=self, args=args)(raw) + return self.to(exploded, of_type, factory) def found_keys(self) -> set[str]: return set(self.content.keys()) - self._unused_exclude @@ -69,7 +80,7 @@ def to_list(value: TomlTypes, of_type: type[_T]) -> Iterator[_T]: @staticmethod def to_set(value: TomlTypes, of_type: type[_T]) -> Iterator[_T]: - of = Set[of_type] # type: ignore[valid-type] # no mypy support + of = List[of_type] # type: ignore[valid-type] # no mypy support return iter(validate(value, of)) # type: ignore[call-overload,no-any-return] @staticmethod diff --git a/src/tox/config/loader/toml/_replace.py b/src/tox/config/loader/toml/_replace.py index f968d23b3..55ce12d95 100644 --- a/src/tox/config/loader/toml/_replace.py +++ b/src/tox/config/loader/toml/_replace.py @@ -1,54 +1,147 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Final +import re +from typing import TYPE_CHECKING, Any, Iterator, List, cast + +from tox.config.loader.replacer import MatchRecursionError, ReplaceReference, load_posargs, replace, replace_env +from tox.config.loader.stringify import stringify + +from ._api import TomlTypes +from ._validate import validate if TYPE_CHECKING: from tox.config.loader.api import ConfigLoadArgs from tox.config.loader.toml import TomlLoader from tox.config.main import Config + from tox.config.sets import ConfigSet + from tox.config.source.toml_pyproject import TomlSection + + +class Unroll: + def __init__(self, conf: Config | None, loader: TomlLoader, args: ConfigLoadArgs) -> None: + self.conf = conf + self.loader = loader + self.args = args + + def __call__(self, value: TomlTypes, depth: int = 0) -> TomlTypes: # noqa: C901, PLR0912 + """Replace all active tokens within value according to the config.""" + depth += 1 + MatchRecursionError.check(depth, value) + if isinstance(value, str): + if self.conf is not None: # core config does not support string substitution + reference = TomlReplaceLoader(self.conf, self.loader) + value = replace(self.conf, reference, value, self.args) + elif isinstance(value, (int, float, bool)): + pass # no reference or substitution possible + elif isinstance(value, list): + # need to inspect every entry of the list to check for reference. + res_list: list[TomlTypes] = [] + for val in value: # apply replacement for every entry + got = self(val, depth) + if isinstance(val, dict) and val.get("replace") in {"posargs", "ref"} and isinstance(got, (list, set)): + res_list.extend(got) + else: + res_list.append(got) + value = res_list + elif isinstance(value, dict): + # need to inspect every entry of the list to check for reference. + if replace_type := value.get("replace"): + if replace_type == "posargs" and self.conf is not None: + got_posargs = load_posargs(self.conf, self.args) + return ( + [self(v, depth) for v in cast(List[str], value.get("default", []))] + if got_posargs is None + else list(got_posargs) + ) + if replace_type == "env": + return replace_env( + self.conf, + [ + cast(str, validate(value["name"], str)), + cast(str, validate(self(value.get("default", ""), depth), str)), + ], + self.args, + ) + if replace_type == "ref": + if of := value.get("raw"): + validated_of = cast(List[str], validate(of, List[str])) + return self.loader.load_raw_from_root(self.loader.section.SEP.join(validated_of)) + if self.conf is not None: # noqa: SIM102 + if (env := value.get("env")) and (key := value.get("key")): + return cast(TomlTypes, self.conf.get_env(cast(str, env))[cast(str, key)]) + res_dict: dict[str, TomlTypes] = {} + for key, val in value.items(): # apply replacement for every entry + res_dict[key] = self(val, depth) + value = res_dict + return value + + +_REFERENCE_PATTERN = re.compile( + r""" + (\[(?P
.*)])? # default value + (?P[-a-zA-Z0-9_]+) # key + (:(?P.*))? # default value + $ +""", + re.VERBOSE, +) + + +class TomlReplaceLoader(ReplaceReference): + def __init__(self, conf: Config, loader: TomlLoader) -> None: + self.conf = conf + self.loader = loader + + def __call__(self, value: str, conf_args: ConfigLoadArgs) -> str | None: + if match := _REFERENCE_PATTERN.search(value): + settings = match.groupdict() + exception: Exception | None = None + try: + for src in self._config_value_sources(settings["section"], conf_args.env_name): + try: + value = src.load(settings["key"], conf_args.chain) + except KeyError as exc: # if fails, keep trying maybe another source can satisfy # noqa: PERF203 + exception = exc + else: + return stringify(value)[0] + except Exception as exc: # noqa: BLE001 + exception = exc + if exception is not None: + if isinstance(exception, KeyError): # if the lookup failed replace - else keep + default = settings["default"] + if default is not None: + return default + raise exception + return value + + def _config_value_sources(self, sec: str | None, current_env: str | None) -> Iterator[ConfigSet | RawLoader]: + if sec is None: + if current_env is not None: + yield self.conf.get_env(current_env) + yield self.conf.core + return + + section: TomlSection = self.loader.section # type: ignore[assignment] + core_prefix = section.core_prefix() + env_prefix = section.env_prefix() + if sec.startswith(env_prefix): + env = sec[len(env_prefix) + len(section.SEP) :] + yield self.conf.get_env(env) + else: + yield RawLoader(self.loader, sec) + if sec == core_prefix: + yield self.conf.core # try via registered configs + + +class RawLoader: + def __init__(self, loader: TomlLoader, section: str) -> None: + self._loader = loader + self._section = section - from ._api import TomlTypes - -MAX_REPLACE_DEPTH: Final[int] = 100 - - -class MatchRecursionError(ValueError): - """Could not stabilize on replacement value.""" - - -def unroll_refs_and_apply_substitutions( - conf: Config | None, - loader: TomlLoader, - value: TomlTypes, - args: ConfigLoadArgs, - depth: int = 0, -) -> TomlTypes: - """Replace all active tokens within value according to the config.""" - if depth > MAX_REPLACE_DEPTH: - msg = f"Could not expand {value} after recursing {depth} frames" - raise MatchRecursionError(msg) - - if isinstance(value, str): - pass # apply string substitution here - elif isinstance(value, (int, float, bool)): - pass # no reference or substitution possible - elif isinstance(value, list): - # need to inspect every entry of the list to check for reference. - res_list: list[TomlTypes] = [] - for val in value: # apply replacement for every entry - got = unroll_refs_and_apply_substitutions(conf, loader, val, args, depth + 1) - res_list.append(got) - value = res_list - elif isinstance(value, dict): - # need to inspect every entry of the list to check for reference. - res_dict: dict[str, TomlTypes] = {} - for key, val in value.items(): # apply replacement for every entry - got = unroll_refs_and_apply_substitutions(conf, loader, val, args, depth + 1) - res_dict[key] = got - value = res_dict - return value + def load(self, item: str, chain: list[str] | None = None) -> Any: # noqa: ARG002 + return self._loader.load_raw_from_root(f"{self._section}{self._loader.section.SEP}{item}") __all__ = [ - "unroll_refs_and_apply_substitutions", + "Unroll", ] diff --git a/src/tox/config/loader/toml/_validate.py b/src/tox/config/loader/toml/_validate.py index e2f5534d7..590d35578 100644 --- a/src/tox/config/loader/toml/_validate.py +++ b/src/tox/config/loader/toml/_validate.py @@ -7,7 +7,6 @@ Dict, List, Literal, - Set, TypeVar, Union, cast, @@ -41,13 +40,6 @@ def validate(val: TomlTypes, of_type: type[T]) -> TypeGuard[T]: # noqa: C901, P elif isclass(of_type) and issubclass(of_type, Command): # first we cast it to list then create commands, so for now validate it as a nested list validate(val, List[str]) - elif casting_to in {set, Set}: - entry_type = of_type.__args__[0] # type: ignore[attr-defined] - if isinstance(val, set): - for va in val: - validate(va, entry_type) - else: - msg = f"{val!r} is not set" elif casting_to in {dict, Dict}: key_type, value_type = of_type.__args__[0], of_type.__args__[1] # type: ignore[attr-defined] if isinstance(val, dict): diff --git a/src/tox/config/of_type.py b/src/tox/config/of_type.py index 27752ef0f..fd978bb4a 100644 --- a/src/tox/config/of_type.py +++ b/src/tox/config/of_type.py @@ -13,6 +13,10 @@ from tox.config.main import Config # pragma: no cover +class CircularChainError(ValueError): + """circular chain in config""" + + T = TypeVar("T") V = TypeVar("V") @@ -94,11 +98,13 @@ def __call__( if self._cache is _PLACE_HOLDER: for key, loader in product(self.keys, loaders): chain_key = f"{loader.section.key}.{key}" - if chain_key in args.chain: - values = args.chain[args.chain.index(chain_key) :] - msg = f"circular chain detected {', '.join(values)}" - raise ValueError(msg) - args.chain.append(chain_key) + try: + if chain_key in args.chain: + values = args.chain[args.chain.index(chain_key) :] + msg = f"circular chain detected {', '.join(values)}" + raise CircularChainError(msg) + finally: + args.chain.append(chain_key) try: value = loader.load(key, self.of_type, self.factory, conf, args) except KeyError: diff --git a/src/tox/config/set_env.py b/src/tox/config/set_env.py index 47fb62238..e3f012ffd 100644 --- a/src/tox/config/set_env.py +++ b/src/tox/config/set_env.py @@ -10,7 +10,7 @@ class SetEnv: - def __init__(self, raw: str, name: str, env_name: str | None, root: Path) -> None: + def __init__(self, raw: str | dict[str, str], name: str, env_name: str | None, root: Path) -> None: self.changed = False self._materialized: dict[str, str] = {} # env vars we already loaded self._raw: dict[str, str] = {} # could still need replacement @@ -18,8 +18,11 @@ def __init__(self, raw: str, name: str, env_name: str | None, root: Path) -> Non self._env_files: list[str] = [] self._replacer: Replacer = lambda s, c: s # noqa: ARG005 self._name, self._env_name, self._root = name, env_name, root - from .loader.ini.replace import MatchExpression, find_replace_expr # noqa: PLC0415 + from .loader.replacer import MatchExpression, find_replace_expr # noqa: PLC0415 + if isinstance(raw, dict): + self._raw = raw + return for line in raw.splitlines(): # noqa: PLR1702 if line.strip(): if line.startswith("file|"): diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index ebb46be69..5182043e1 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -232,7 +232,10 @@ def set_env_post_process(values: SetEnv) -> SetEnv: return values def set_env_factory(raw: object) -> SetEnv: - if not isinstance(raw, str): + if not ( + isinstance(raw, str) + or (isinstance(raw, dict) and all(isinstance(k, str) and isinstance(v, str) for k, v in raw.items())) + ): raise TypeError(raw) return SetEnv(raw, self.name, self.env_name, root) diff --git a/src/tox/config/source/toml_pyproject.py b/src/tox/config/source/toml_pyproject.py index ddf5d6688..4bb915903 100644 --- a/src/tox/config/source/toml_pyproject.py +++ b/src/tox/config/source/toml_pyproject.py @@ -39,6 +39,10 @@ def test_env(cls, name: str) -> TomlSection: def env_prefix(cls) -> str: return cls.SEP.join((*cls.PREFIX, cls.ENV)) + @classmethod + def core_prefix(cls) -> str: + return cls.SEP.join(cls.PREFIX) + @classmethod def package_env_base(cls) -> str: return cls.SEP.join((*cls.PREFIX, cls.PKG_ENV_BASE)) @@ -49,7 +53,11 @@ def run_env_base(cls) -> str: @property def keys(self) -> Iterable[str]: - return self.key.split(self.SEP) if self.key else [] + key = self.key + keys = key.split(self.SEP) if self.key else [] + if self.PREFIX and len(keys) >= len(self.PREFIX) and tuple(keys[: len(self.PREFIX)]) == self.PREFIX: + keys = keys[len(self.PREFIX) :] + return keys class TomlPyProjectSection(TomlSection): @@ -66,19 +74,19 @@ def __init__(self, path: Path) -> None: if path.name != self.FILENAME or not path.exists(): raise ValueError with path.open("rb") as file_handler: - toml_content = tomllib.load(file_handler) + self._content = tomllib.load(file_handler) + our_content: Mapping[str, Any] = self._content + for key in self._Section.PREFIX: + our_content = our_content[key] + self._our_content = our_content try: - content: Mapping[str, Any] = toml_content - for key in self._Section.PREFIX: - content = content[key] - self._content = content self._post_validate() except KeyError as exc: raise ValueError(path) from exc super().__init__(path) def _post_validate(self) -> None: - if "legacy_tox_ini" in self._content: + if "legacy_tox_ini" in self._our_content: msg = "legacy_tox_ini" raise KeyError(msg) @@ -89,7 +97,7 @@ def transform_section(self, section: Section) -> Section: return self._Section(section.prefix, section.name) def get_loader(self, section: Section, override_map: OverrideMap) -> Loader[Any] | None: - current = self._content + current = self._our_content sec = cast(TomlSection, section) for key in sec.keys: if key in current: @@ -103,6 +111,7 @@ def get_loader(self, section: Section, override_map: OverrideMap) -> Loader[Any] section=section, overrides=override_map.get(section.key, []), content=current, + root_content=self._content, unused_exclude={sec.ENV, sec.RUN_ENV_BASE, sec.PKG_ENV_BASE} if section.prefix is None else set(), ) @@ -111,7 +120,7 @@ def envs(self, core_conf: CoreConfigSet) -> Iterator[str]: yield from [i.key for i in self.sections()] def sections(self) -> Iterator[Section]: - for env_name in self._content.get(self._Section.ENV, {}): + for env_name in self._our_content.get(self._Section.ENV, {}): yield self._Section.from_key(env_name) def get_base_sections(self, base: list[str], in_section: Section) -> Iterator[Section]: # noqa: ARG002 diff --git a/tests/config/loader/ini/replace/conftest.py b/tests/config/loader/conftest.py similarity index 100% rename from tests/config/loader/ini/replace/conftest.py rename to tests/config/loader/conftest.py diff --git a/tests/config/loader/ini/replace/test_replace_tox_env.py b/tests/config/loader/ini/replace/test_replace_tox_env.py index 7d89d8cbd..bc5d866ef 100644 --- a/tests/config/loader/ini/replace/test_replace_tox_env.py +++ b/tests/config/loader/ini/replace/test_replace_tox_env.py @@ -5,7 +5,7 @@ import pytest -from tox.config.loader.ini.replace import MAX_REPLACE_DEPTH +from tox.config.loader.replacer import MAX_REPLACE_DEPTH from tox.config.sets import ConfigSet from tox.report import HandledError diff --git a/tests/config/loader/ini/replace/test_replace.py b/tests/config/loader/test_replace.py similarity index 97% rename from tests/config/loader/ini/replace/test_replace.py rename to tests/config/loader/test_replace.py index 7b203adf2..c2645f189 100644 --- a/tests/config/loader/ini/replace/test_replace.py +++ b/tests/config/loader/test_replace.py @@ -4,7 +4,7 @@ import pytest -from tox.config.loader.ini.replace import MatchExpression, find_replace_expr +from tox.config.loader.replacer import MatchExpression, find_replace_expr from tox.report import HandledError if TYPE_CHECKING: diff --git a/tests/config/loader/test_toml_loader.py b/tests/config/loader/test_toml_loader.py index 159c4b709..670edac3e 100644 --- a/tests/config/loader/test_toml_loader.py +++ b/tests/config/loader/test_toml_loader.py @@ -1,7 +1,7 @@ from __future__ import annotations from pathlib import Path -from typing import Any, Dict, List, Literal, Optional, Set, TypeVar +from typing import Any, Dict, List, Literal, Optional, TypeVar import pytest @@ -12,17 +12,17 @@ def test_toml_loader_load_raw() -> None: - loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1, "c": False}, set()) + loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1, "c": False}, {}, set()) assert loader.load_raw("a", None, "A") == 1 def test_toml_loader_load_repr() -> None: - loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1}, set()) + loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1}, {}, set()) assert repr(loader) == "TomlLoader(env.A, {'a': 1})" def test_toml_loader_found_keys() -> None: - loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1, "c": False}, set()) + loader = TomlLoader(TomlPyProjectSection.from_key("tox.env.A"), [], {"a": 1, "c": False}, {}, set()) assert loader.found_keys() == {"a", "c"} @@ -35,7 +35,7 @@ def factory_na(obj: object) -> None: def perform_load(value: Any, of_type: type[V]) -> V: env_name, key = "A", "k" - loader = TomlLoader(TomlPyProjectSection.from_key(f"tox.env.{env_name}"), [], {key: value}, set()) + loader = TomlLoader(TomlPyProjectSection.from_key(f"tox.env.{env_name}"), [], {key: value}, {}, set()) args = ConfigLoadArgs(None, env_name, env_name) return loader.load(key, of_type, factory_na, None, args) # type: ignore[arg-type] @@ -72,20 +72,6 @@ def test_toml_loader_list_nok_element() -> None: perform_load(["a", 2], List[str]) -def test_toml_loader_set_ok() -> None: - assert perform_load({"a"}, Set[str]) == {"a"} - - -def test_toml_loader_set_nok() -> None: - with pytest.raises(TypeError, match="{} is not set"): - perform_load({}, Set[str]) - - -def test_toml_loader_set_nok_element() -> None: - with pytest.raises(TypeError, match="2 is not of type 'str'"): - perform_load({"a", 2}, Set[str]) - - def test_toml_loader_dict_ok() -> None: assert perform_load({"a": "1"}, Dict[str, str]) == {"a": "1"} diff --git a/tests/config/source/test_toml_pyproject.py b/tests/config/source/test_toml_pyproject.py index f3ec1552f..cd6b81a09 100644 --- a/tests/config/source/test_toml_pyproject.py +++ b/tests/config/source/test_toml_pyproject.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import pytest + from tox.pytest import ToxProjectCreator @@ -60,3 +62,175 @@ def test_config_in_toml_extra(tox_project: ToxProjectCreator) -> None: outcome.assert_success() assert "# Exception: " not in outcome.out, outcome.out assert "# !!! unused: " not in outcome.out, outcome.out + + +def test_config_in_toml_replace_default(tox_project: ToxProjectCreator) -> None: + project = tox_project({"pyproject.toml": '[tool.tox.env_run_base]\ndescription = "{missing:miss}"'}) + outcome = project.run("c", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = miss\n", "") + + +def test_config_in_toml_replace_env_name_via_env(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": '[tool.tox.env_run_base]\ndescription = "Magic in {env:MAGICAL:{env_name}}"' + }) + outcome = project.run("c", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = Magic in py\n", "") + + +def test_config_in_toml_replace_env_name_via_env_set( + tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MAGICAL", "YEAH") + project = tox_project({ + "pyproject.toml": '[tool.tox.env_run_base]\ndescription = "Magic in {env:MAGICAL:{env_name}}"' + }) + outcome = project.run("c", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = Magic in YEAH\n", "") + + +def test_config_in_toml_replace_from_env_section_absolute(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.A] + description = "a" + [tool.tox.env.B] + description = "{[tool.tox.env.A]env_name}" + """ + }) + outcome = project.run("c", "-e", "B", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:B]\ndescription = A\n", "") + + +def test_config_in_toml_replace_from_section_absolute(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.extra] + ok = "o" + [tool.tox.env.B] + description = "{[tool.tox.extra]ok}" + """ + }) + outcome = project.run("c", "-e", "B", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:B]\ndescription = o\n", "") + + +def test_config_in_toml_replace_posargs_default(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.A] + commands = [["python", { replace = "posargs", default = ["a", "b"] } ]] + """ + }) + outcome = project.run("c", "-e", "A", "-k", "commands") + outcome.assert_success() + outcome.assert_out_err("[testenv:A]\ncommands = python a b\n", "") + + +def test_config_in_toml_replace_posargs_set(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.A] + commands = [["python", { replace = "posargs", default = ["a", "b"] } ]] + """ + }) + outcome = project.run("c", "-e", "A", "-k", "commands", "--", "c", "d") + outcome.assert_success() + outcome.assert_out_err("[testenv:A]\ncommands = python c d\n", "") + + +def test_config_in_toml_replace_env_default(tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.A] + description = { replace = "env", name = "NAME", default = "OK" } + """ + }) + monkeypatch.delenv("NAME", raising=False) + + outcome = project.run("c", "-e", "A", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:A]\ndescription = OK\n", "") + + +def test_config_in_toml_replace_env_set(tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.A] + description = { replace = "env", name = "NAME", default = "OK" } + """ + }) + monkeypatch.setenv("NAME", "OK2") + + outcome = project.run("c", "-e", "A", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:A]\ndescription = OK2\n", "") + + +def test_config_in_toml_replace_ref_raw(tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env_run_base] + extras = ["A", "{env_name}"] + [tool.tox.env.a] + extras = [{ replace = "ref", raw = ["tool", "tox", "env_run_base", "extras"] }, "B"] + """ + }) + monkeypatch.setenv("NAME", "OK2") + + outcome = project.run("c", "-e", "a", "-k", "extras") + outcome.assert_success() + outcome.assert_out_err("[testenv:a]\nextras =\n a\n b\n {env-name}\n", "") + + +def test_config_in_toml_replace_ref_env(tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.b] + extras = ["{env_name}"] + [tool.tox.env.a] + extras = [{ replace = "ref", env = "b", "key" = "extras" }, "a"] + """ + }) + monkeypatch.setenv("NAME", "OK2") + + outcome = project.run("c", "-e", "a", "-k", "extras") + outcome.assert_success() + outcome.assert_out_err("[testenv:a]\nextras =\n a\n b\n", "") + + +def test_config_in_toml_replace_env_circular_set( + tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch +) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.a] + set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{env_name}" } + """ + }) + monkeypatch.setenv("COVERAGE_FILE", "OK") + + outcome = project.run("c", "-e", "a", "-k", "set_env") + outcome.assert_success() + assert "COVERAGE_FILE=OK" in outcome.out, outcome.out + + +def test_config_in_toml_replace_env_circular_unset( + tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch +) -> None: + project = tox_project({ + "pyproject.toml": """ + [tool.tox.env.a] + set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{env_name}" } + """ + }) + monkeypatch.delenv("COVERAGE_FILE", raising=False) + + outcome = project.run("c", "-e", "a", "-k", "set_env") + outcome.assert_success() + assert "COVERAGE_FILE=a" in outcome.out, outcome.out diff --git a/tests/config/source/test_toml_tox.py b/tests/config/source/test_toml_tox.py index 742393df2..7c3c24a08 100644 --- a/tests/config/source/test_toml_tox.py +++ b/tests/config/source/test_toml_tox.py @@ -4,6 +4,8 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: + import pytest + from tox.pytest import ToxProjectCreator @@ -61,7 +63,53 @@ def test_config_in_toml_extra(tox_project: ToxProjectCreator) -> None: assert "# !!! unused: " not in outcome.out, outcome.out -def test_config_in_toml_replace(tox_project: ToxProjectCreator) -> None: - project = tox_project({"tox.toml": '[env_run_base]\ndescription = "Magic in {env_name}"'}) +def test_config_in_toml_replace_default(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": '[env_run_base]\ndescription = "{missing:miss}"'}) + outcome = project.run("c", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = miss\n", "") + + +def test_config_in_toml_replace_env_name_via_env(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": '[env_run_base]\ndescription = "Magic in {env:MAGICAL:{env_name}}"'}) outcome = project.run("c", "-k", "description") outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = Magic in py\n", "") + + +def test_config_in_toml_replace_env_name_via_env_set( + tox_project: ToxProjectCreator, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("MAGICAL", "YEAH") + project = tox_project({"tox.toml": '[env_run_base]\ndescription = "Magic in {env:MAGICAL:{env_name}}"'}) + outcome = project.run("c", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:py]\ndescription = Magic in YEAH\n", "") + + +def test_config_in_toml_replace_from_env_section_absolute(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "tox.toml": """ + [env.A] + description = "a" + [env.B] + description = "{[env.A]env_name}" + """ + }) + outcome = project.run("c", "-e", "B", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:B]\ndescription = A\n", "") + + +def test_config_in_toml_replace_from_section_absolute(tox_project: ToxProjectCreator) -> None: + project = tox_project({ + "tox.toml": """ + [extra] + ok = "o" + [env.B] + description = "{[extra]ok}" + """ + }) + outcome = project.run("c", "-e", "B", "-k", "description") + outcome.assert_success() + outcome.assert_out_err("[testenv:B]\ndescription = o\n", "") diff --git a/tox.toml b/tox.toml index f44cca119..4d07a37b2 100644 --- a/tox.toml +++ b/tox.toml @@ -8,12 +8,12 @@ package = "wheel" wheel_build_env = ".pkg" extras = ["testing"] pass_env = ["PYTEST_*", "SSL_CERT_FILE"] -set_env.COVERAGE_FILE = { type = "env", name = "COVERAGE_FILE", default = "{work_dir}{/}.coverage.{env_name}" } +set_env.COVERAGE_FILE = { replace = "env", name = "COVERAGE_FILE", default = "{work_dir}{/}.coverage.{env_name}" } set_env.COVERAGE_FILECOVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml" commands = [ [ "pytest", - { type = "posargs", default = [ + { replace = "posargs", default = [ "--junitxml", "{work_dir}{/}junit.{env_name}.xml", "--cov", @@ -30,7 +30,7 @@ commands = [ "--cov-report", "xml:{work_dir}{/}coverage.{env_name}.xml", "-n", - { type = "env", name = "PYTEST_XDIST_AUTO_NUM_WORKERS", default = "auto" }, + { replace = "env", name = "PYTEST_XDIST_AUTO_NUM_WORKERS", default = "auto" }, "tests", "--durations", "15", @@ -40,7 +40,7 @@ commands = [ [ "diff-cover", "--compare-branch", - { type = "env", name = "DIFF_AGAINST", default = "origin/main" }, + { replace = "env", name = "DIFF_AGAINST", default = "origin/main" }, "{work_dir}{/}coverage.{env_name}.xml", ], ] @@ -49,19 +49,19 @@ commands = [ description = "format the code base to adhere to our styles, and complain about what we cannot do automatically" skip_install = true deps = ["pre-commit-uv>=4.1.3"] -pass_env = [{ type = "ref", of = ["env_run_base", "pass_env"] }, "PROGRAMDATA"] -commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure", { type = "posargs" }]] +pass_env = [{ replace = "ref", of = ["env_run_base", "pass_env"] }, "PROGRAMDATA"] +commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure", { replace = "posargs" }]] [env.type] description = "run type check on code base" deps = ["mypy==1.11.2", "types-cachetools>=5.5.0.20240820", "types-chardet>=5.0.4.6"] -commands = [["mypy", "src/tox"], ["mypy", "tests"]] +commands = [["mypy", "src{/}tox"], ["mypy", "tests"]] [env.docs] description = "build documentation" extras = ["docs"] commands = [ - { type = "posargs", default = [ + { replace = "posargs", default = [ "sphinx-build", "-d", "{env_tmp_dir}{/}docs_tree", @@ -125,7 +125,7 @@ commands = [["python", "{tox_root}/tasks/release.py", "--version", "{posargs}"]] [env.dev] description = "dev environment with all deps at {envdir}" package = "editable" -deps = { type = "ref", of = ["env", "release", "deps"] } +deps = { replace = "ref", of = ["env", "release", "deps"] } extras = ["docs", "testing"] commands = [["python", "-m", "pip", "list", "--format=columns"], ["python", "-c", 'print(r"{env_python}")']] uv_seed = true