From 7ebf6a3db6761bb36cfbd27f89cab45f87007eeb Mon Sep 17 00:00:00 2001 From: jeanluc Date: Tue, 7 Nov 2023 00:02:30 +0100 Subject: [PATCH] Add `defaults` SSH wrapper module This is a 1:1 copy of the execution module, incl. tests... --- changelog/51605.fixed.md | 1 + salt/client/ssh/wrapper/defaults.py | 240 ++++++++++++++++++ .../unit/client/ssh/wrapper/test_defaults.py | 215 ++++++++++++++++ 3 files changed, 456 insertions(+) create mode 100644 changelog/51605.fixed.md create mode 100644 salt/client/ssh/wrapper/defaults.py create mode 100644 tests/pytests/unit/client/ssh/wrapper/test_defaults.py diff --git a/changelog/51605.fixed.md b/changelog/51605.fixed.md new file mode 100644 index 000000000000..990b34413d95 --- /dev/null +++ b/changelog/51605.fixed.md @@ -0,0 +1 @@ +Fixed defaults.merge is not available when using salt-ssh diff --git a/salt/client/ssh/wrapper/defaults.py b/salt/client/ssh/wrapper/defaults.py new file mode 100644 index 000000000000..d03990b87980 --- /dev/null +++ b/salt/client/ssh/wrapper/defaults.py @@ -0,0 +1,240 @@ +""" +SSH wrapper module to work with salt formula defaults files + +""" + +import copy +import logging +import os + +import salt.fileclient +import salt.utils.data +import salt.utils.dictupdate as dictupdate +import salt.utils.files +import salt.utils.json +import salt.utils.url +import salt.utils.yaml + +__virtualname__ = "defaults" + +log = logging.getLogger(__name__) + + +def _mk_client(): + """ + Create a file client and add it to the context + """ + return salt.fileclient.get_file_client(__opts__) + + +def _load(formula): + """ + Generates a list of salt:///defaults.(json|yaml) files + and fetches them from the Salt master. + + Returns first defaults file as python dict. + """ + + # Compute possibilities + paths = [] + for ext in ("yaml", "json"): + source_url = salt.utils.url.create(formula + "/defaults." + ext) + paths.append(source_url) + # Fetch files from master + with _mk_client() as client: + defaults_files = client.cache_files(paths) + + for file_ in defaults_files: + if not file_: + # Skip empty string returned by cp.fileclient.cache_files. + continue + + suffix = file_.rsplit(".", 1)[-1] + if suffix == "yaml": + loader = salt.utils.yaml.safe_load + elif suffix == "json": + loader = salt.utils.json.load + else: + log.debug("Failed to determine loader for %r", file_) + continue + + if os.path.exists(file_): + log.debug("Reading defaults from %r", file_) + with salt.utils.files.fopen(file_) as fhr: + defaults = loader(fhr) + log.debug("Read defaults %r", defaults) + + return defaults or {} + + +def get(key, default=""): + """ + defaults.get is used much like pillar.get except that it will read + a default value for a pillar from defaults.json or defaults.yaml + files that are stored in the root of a salt formula. + + CLI Example: + + .. code-block:: bash + + salt '*' defaults.get core:users:root + + The defaults is computed from pillar key. The first entry is considered as + the formula namespace. + + For example, querying ``core:users:root`` will try to load + ``salt://core/defaults.yaml`` and ``salt://core/defaults.json``. + """ + + # Determine formula namespace from query + if ":" in key: + namespace, key = key.split(":", 1) + else: + namespace, key = key, None + + # Fetch and load defaults formula files from states. + defaults = _load(namespace) + + # Fetch value + if key: + return salt.utils.data.traverse_dict_and_list(defaults, key, default) + else: + return defaults + + +def merge(dest, src, merge_lists=False, in_place=True, convert_none=True): + """ + defaults.merge + Allows deep merging of dicts in formulas. + + merge_lists : False + If True, it will also merge lists instead of replace their items. + + in_place : True + If True, it will merge into dest dict, + if not it will make a new copy from that dict and return it. + + convert_none : True + If True, it will convert src and dest to empty dicts if they are None. + If True and dest is None but in_place is True, raises TypeError. + If False it will make a new copy from that dict and return it. + + .. versionadded:: 3005 + + CLI Example: + + .. code-block:: bash + + salt '*' defaults.merge '{a: b}' '{d: e}' + + It is more typical to use this in a templating language in formulas, + instead of directly on the command-line. + """ + # Force empty dicts if applicable (useful for cleaner templating) + src = {} if (src is None and convert_none) else src + if dest is None and convert_none: + if in_place: + raise TypeError("Can't perform in-place merge into NoneType") + else: + dest = {} + + if in_place: + merged = dest + else: + merged = copy.deepcopy(dest) + return dictupdate.update(merged, src, merge_lists=merge_lists) + + +def deepcopy(source): + """ + defaults.deepcopy + Allows deep copy of objects in formulas. + + By default, Python does not copy objects, + it creates bindings between a target and an object. + + It is more typical to use this in a templating language in formulas, + instead of directly on the command-line. + """ + return copy.deepcopy(source) + + +def update(dest, defaults, merge_lists=True, in_place=True, convert_none=True): + """ + defaults.update + Allows setting defaults for group of data set e.g. group for nodes. + + This function is a combination of defaults.merge + and defaults.deepcopy to avoid redundant in jinja. + + Example: + + .. code-block:: yaml + + group01: + defaults: + enabled: True + extra: + - test + - stage + nodes: + host01: + index: foo + upstream: bar + host02: + index: foo2 + upstream: bar2 + + .. code-block:: jinja + + {% do salt['defaults.update'](group01.nodes, group01.defaults) %} + + Each node will look like the following: + + .. code-block:: yaml + + host01: + enabled: True + index: foo + upstream: bar + extra: + - test + - stage + + merge_lists : True + If True, it will also merge lists instead of replace their items. + + in_place : True + If True, it will merge into dest dict. + if not it will make a new copy from that dict and return it. + + convert_none : True + If True, it will convert src and dest to empty dicts if they are None. + If True and dest is None but in_place is True, raises TypeError. + If False it will make a new copy from that dict and return it. + + .. versionadded:: 3005 + + It is more typical to use this in a templating language in formulas, + instead of directly on the command-line. + """ + # Force empty dicts if applicable here + if in_place: + if dest is None: + raise TypeError("Can't perform in-place update into NoneType") + else: + nodes = dest + else: + dest = {} if (dest is None and convert_none) else dest + nodes = deepcopy(dest) + + defaults = {} if (defaults is None and convert_none) else defaults + + for node_name, node_vars in nodes.items(): + defaults_vars = deepcopy(defaults) + node_vars = merge( + defaults_vars, node_vars, merge_lists=merge_lists, convert_none=convert_none + ) + nodes[node_name] = node_vars + + return nodes diff --git a/tests/pytests/unit/client/ssh/wrapper/test_defaults.py b/tests/pytests/unit/client/ssh/wrapper/test_defaults.py new file mode 100644 index 000000000000..12d07bc2a854 --- /dev/null +++ b/tests/pytests/unit/client/ssh/wrapper/test_defaults.py @@ -0,0 +1,215 @@ +""" +Test cases for salt.client.ssh.wrapper.defaults + +This has been copied 1:1 from tests.pytests.unit.modules.test_defaults +""" + +import inspect + +import pytest + +import salt.client.ssh.wrapper.defaults as defaults +from tests.support.mock import MagicMock, patch + + +@pytest.fixture() +def configure_loader_modules(): + return {defaults: {}} + + +def test_get_mock(): + """ + Test if it execute a defaults client run and return a dict + """ + with patch.object(inspect, "stack", MagicMock(return_value=[])), patch( + "salt.client.ssh.wrapper.defaults.get", + MagicMock(return_value={"users": {"root": [0]}}), + ): + assert defaults.get("core:users:root") == {"users": {"root": [0]}} + + +def test_merge_with_list_merging(): + """ + Test deep merging of dicts with merge_lists enabled. + """ + + src_dict = { + "string_key": "string_val_src", + "list_key": ["list_val_src"], + "dict_key": {"dict_key_src": "dict_val_src"}, + } + + dest_dict = { + "string_key": "string_val_dest", + "list_key": ["list_val_dest"], + "dict_key": {"dict_key_dest": "dict_val_dest"}, + } + + merged_dict = { + "string_key": "string_val_src", + "list_key": ["list_val_dest", "list_val_src"], + "dict_key": { + "dict_key_dest": "dict_val_dest", + "dict_key_src": "dict_val_src", + }, + } + + defaults.merge(dest_dict, src_dict, merge_lists=True) + assert dest_dict == merged_dict + + +def test_merge_without_list_merging(): + """ + Test deep merging of dicts with merge_lists disabled. + """ + + src = { + "string_key": "string_val_src", + "list_key": ["list_val_src"], + "dict_key": {"dict_key_src": "dict_val_src"}, + } + + dest = { + "string_key": "string_val_dest", + "list_key": ["list_val_dest"], + "dict_key": {"dict_key_dest": "dict_val_dest"}, + } + + merged = { + "string_key": "string_val_src", + "list_key": ["list_val_src"], + "dict_key": { + "dict_key_dest": "dict_val_dest", + "dict_key_src": "dict_val_src", + }, + } + + defaults.merge(dest, src, merge_lists=False) + assert dest == merged + + +def test_merge_not_in_place(): + """ + Test deep merging of dicts not in place. + """ + + src = {"nested_dict": {"A": "A"}} + + dest = {"nested_dict": {"B": "B"}} + + dest_orig = {"nested_dict": {"B": "B"}} + + merged = {"nested_dict": {"A": "A", "B": "B"}} + + final = defaults.merge(dest, src, in_place=False) + assert dest == dest_orig + assert final == merged + + +def test_merge_src_is_none(): + """ + Test deep merging of dicts not in place. + """ + + dest = {"nested_dict": {"B": "B"}} + + dest_orig = {"nested_dict": {"B": "B"}} + + final = defaults.merge(dest, None, in_place=False) + assert dest == dest_orig + assert final == dest_orig + + +def test_merge_dest_is_none(): + """ + Test deep merging of dicts not in place. + """ + + src = {"nested_dict": {"B": "B"}} + + src_orig = {"nested_dict": {"B": "B"}} + + final = defaults.merge(None, src, in_place=False) + assert src == src_orig + assert final == src_orig + + +def test_merge_in_place_dest_is_none(): + """ + Test deep merging of dicts not in place. + """ + + src = {"nested_dict": {"B": "B"}} + + pytest.raises(TypeError, defaults.merge, None, src) + + +def test_deepcopy(): + """ + Test a deep copy of object. + """ + + src = {"A": "A", "B": "B"} + + dist = defaults.deepcopy(src) + dist.update({"C": "C"}) + + result = {"A": "A", "B": "B", "C": "C"} + + assert src != dist + assert dist == result + + +def test_update_in_place(): + """ + Test update with defaults values in place. + """ + + group01 = { + "defaults": {"enabled": True, "extra": ["test", "stage"]}, + "nodes": {"host01": {"index": "foo", "upstream": "bar"}}, + } + + host01 = { + "enabled": True, + "index": "foo", + "upstream": "bar", + "extra": ["test", "stage"], + } + + defaults.update(group01["nodes"], group01["defaults"]) + assert group01["nodes"]["host01"] == host01 + + +def test_update_with_defaults_none(): + group01 = { + "defaults": {"enabled": True, "extra": ["test", "stage"]}, + "nodes": {"host01": {"index": "foo", "upstream": "bar"}}, + } + + host01 = { + "index": "foo", + "upstream": "bar", + } + + defaults.update(group01["nodes"], None) + assert group01["nodes"]["host01"] == host01 + + +def test_update_with_dest_none(): + group01 = { + "defaults": {"enabled": True, "extra": ["test", "stage"]}, + "nodes": {"host01": {"index": "foo", "upstream": "bar"}}, + } + + ret = defaults.update(None, group01["defaults"], in_place=False) + assert ret == {} + + +def test_update_in_place_with_dest_none(): + group01 = { + "defaults": {"enabled": True, "extra": ["test", "stage"]}, + "nodes": {"host01": {"index": "foo", "upstream": "bar"}}, + } + + pytest.raises(TypeError, defaults.update, None, group01["defaults"])