From ce1782cab01e271993fb45df3e98928ae58e7b35 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 00:02:40 +0100 Subject: [PATCH 1/5] feat(map.jinja): load a configurable list of YAML files Lookup the configuration for `map.jinja` from: 1. builtin default 1. `osarch` grain 2. `os_family` grain 3. `os` grain 4. `osfinger` grain 5. `lookup` table retrived by `config.get` 6. configuration retrived by `config.get` 7. `id` grain 2. `defaults.yaml`: optionally define a formula specific `map_jinja:sources` 3. global configuration lookup `map_jinja:sources` 4. formula specific `:map_jinja:sources` The hard coded default is backward compatible and add the minion id at the end of the list. If an entry does not match a `salt['config.get']` parameter, it's used as a literal file path (with `.yaml` added if it's missing). Each YAML file are formatted with the following top level keys: - `values`: contains the the parameter values - `strategy` (optional): define the merge strategy for the function `salt.slsutils.merge` (aggregate, list, overwrite, recurse, smart). The default is `smart`. - `merge_lists` (optional): boolean to merge lists or overwrite them It's possile to configure `config.get` when looking up the formula configuration. You need to define a subkey `strategy` with the following differences from YAML files: - `strategy` can only be one of `None`, `overwrite` and `recurse`. The default is `None`. - if you use `salt-ssh`, this merge strategy is skipped and and error log message is emitted. The parameters values are merged in the following order: 1. initialize default values from `defaults.yaml` 2. merge the values from each source defined by the ordered list `map_jinja:sources` --- libvirt/map.jinja | 154 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 115 insertions(+), 39 deletions(-) diff --git a/libvirt/map.jinja b/libvirt/map.jinja index d7be626..7eeac1b 100644 --- a/libvirt/map.jinja +++ b/libvirt/map.jinja @@ -3,42 +3,118 @@ {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} -{#- Start imports as #} -{%- import_yaml tplroot ~ "/defaults.yaml" as default_settings %} -{%- import_yaml tplroot ~ "/osfamilymap.yaml" as osfamilymap %} -{%- import_yaml tplroot ~ "/osmap.yaml" as osmap %} -{%- import_yaml tplroot ~ "/osfingermap.yaml" as osfingermap %} - -{#- Retrieve the config dict only once #} -{%- set _config = salt['config.get'](tplroot, default={}) %} - -{%- set defaults = salt['grains.filter_by']( - default_settings, - default=tplroot, - merge=salt['grains.filter_by']( - osfamilymap, - grain='os_family', - merge=salt['grains.filter_by']( - osmap, - grain='os', - merge=salt['grains.filter_by']( - osfingermap, - grain='osfinger', - merge=salt['grains.filter_by']( - _config, - default='lookup' - ) - ) - ) - ) - ) - %} - -{%- set config = salt['grains.filter_by']( - {'defaults': defaults}, - default='defaults', - merge=_config - ) -%} - -{%- set libvirt_settings = config %} + +{#- Where to lookup parameters source files #} +{%- set map_sources_dir = tplroot | path_join('parameters') %} + +{#- Load defaults first to allow per formula default map.jinja configuration #} +{%- set _defaults_filename = map_sources_dir | path_join('defaults.yaml') %} +{%- do salt['log.debug']('map.jinja: initialise parameters from ' ~ _defaults_filename ) %} +{%- import_yaml _defaults_filename as default_settings %} + +{#- List of sources to lookup for parameters #} +{%- do salt['log.debug']("map.jinja: lookup 'map_jinja' configuration sources") %} +{#- Fallback to previously used grains plus minion `id` #} +{%- set map_sources = ['osarch', 'os_family', 'os', 'osfinger', 'config_get_lookup', 'config_get', 'id'] %} +{#- Configure map.jinja from defaults.yaml #} +{%- set map_sources = default_settings | traverse('values:map_jinja:sources', map_sources) %} + +{#- Lookup global sources #} +{%- set map_sources = salt['config.get']('map_jinja:sources', map_sources) %} +{#- Lookup per formula sources #} +{%- set map_sources = salt['config.get'](tplroot ~ ':map_jinja:sources', map_sources) %} + +{%- do salt['log.debug']('map.jinja: load parameters with sources from ' ~ map_sources) %} + + +{#- Work around assignment inside for loop #} +{#- load configuration values used in `config.get` merging strategies #} +{%- set _config = {'stack': default_settings.get('values', {}), + 'merge_strategy': salt['config.get'](tplroot ~ ':strategy', None), + 'merge_lists': salt['config.get'](tplroot ~ ':merge_lists', False), + 'cli': salt['config.get']('__cli'), + 'root_dir': salt['config.get']('root_dir') + } %} + +{#- `config.get` merge option works only for `minion` or local `salt-call` #} +{%- if _config['cli'] == 'salt-minion' + or (_config['cli'] == 'salt-call' and not _config['root_dir'].endswith('/running_data')) %} + {%- do _config.update({'merge_opt': {'merge': _config['merge_strategy']}, + 'merge_msg': ", merge: strategy='" ~ _config['merge_strategy'] ~ "'"}) %} +{%- else %} + {#- no `config.get` merge option for `salt-ssh` or `unknown` cli #} + {%- if _config['merge_strategy'] %} + {%- do salt['log.error']("map.jinja: the 'merge' option of 'config.get' is skipped with salt cli '" + ~ _config['cli'] | replace('call', 'ssh') | default('unknown', True) ~ "'") + %} + {%- endif %} + {%- do _config.update({'merge_opt': {}, + 'merge_msg': ''}) %} +{%- endif %} + + +{#- process each `map.jinja` source #} +{%- for map_source in map_sources %} + {%- if map_source in ['config_get', 'config_get_lookup'] %} + {%- set _config_key = {'config_get': tplroot, + 'config_get_lookup': tplroot ~ ':lookup'}.get(map_source) %} + {%- set _config_type = {'config_get': 'configuration', + 'config_get_lookup': 'lookup'}.get(map_source) %} + + {%- do salt['log.debug']("map.jinja: retrieve formula " ~ _config_type + ~ " with 'config.get'" + ~ _config['merge_msg'] + ) %} + {%- set _config_get = salt['config.get'](_config_key, default={}, **_config['merge_opt']) %} + + {#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #} + {%- set _strategy = _config['merge_strategy'] | default('smart', boolean=True) %} + {%- do salt['log.debug']("map.jinja: merge formula " ~ _config_type + ~ " retrieved with 'config.get'" + ~ ", merge: strategy='" ~ _strategy + ~ "', lists='" ~ _config['merge_lists'] ~ "'" + ) %} + {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'], + _config_get, + strategy=_strategy, + merge_lists=_config['merge_lists'])}) + %} + {%- else %} + {#- Lookup the grain/pillar/... #} + {#- Fallback to use the source name as a direct filename #} + {%- set map_values = salt['config.get'](map_source, []) %} + + {#- Mangle `map_source` to use it as literal path #} + {%- if map_values|length == 0 %} + {%- set map_source_parts = map_source.split('/') %} + {%- set map_source = map_source_parts[0:-1] | join('/') %} + {%- set map_values = map_source_parts[-1].rstrip('.yaml') %} + {%- endif %} + + {#- Some configuration return list #} + {%- if map_values is string %} + {%- set map_values = [map_values] %} + {%- endif %} + + {%- for map_value in map_values %} + {%- set yamlfile = map_sources_dir | path_join(map_source, map_value ~ '.yaml') %} + {%- do salt['log.debug']('map.jinja: load parameters from file ' ~ yamlfile) %} + {%- load_yaml as loaded_values %} + {%- include yamlfile ignore missing %} + {%- endload %} + + {%- if loaded_values %} + {#- Merge loaded values on the stack #} + {%- do salt['log.debug']('map.jinja: merge parameters from ' ~ yamlfile) %} + {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'], + loaded_values.get('values', {}), + strategy=loaded_values.get('strategy', 'smart'), + merge_lists=loaded_values.get('merge_lists', False) | to_bool)}) + %} + {%- endif %} + {%- endfor %} + {%- endif %} +{%- endfor %} + +{%- do salt['log.debug']("map.jinja: save parameters in variable 'libvirt_settings'") %} +{%- set libvirt_settings = _config['stack'] %} From 3ca19bc63b9a631690515df46e07ca4bc7d92807 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 00:03:03 +0100 Subject: [PATCH 2/5] feat(map.jinja): `defaults.yaml` must be under `parameters/` * libvirt/parameters/defaults.yaml: replace the top level key `libvirt` by 'values`. --- libvirt/{ => parameters}/defaults.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename libvirt/{ => parameters}/defaults.yaml (95%) diff --git a/libvirt/defaults.yaml b/libvirt/parameters/defaults.yaml similarity index 95% rename from libvirt/defaults.yaml rename to libvirt/parameters/defaults.yaml index 0822516..98730f1 100644 --- a/libvirt/defaults.yaml +++ b/libvirt/parameters/defaults.yaml @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # vim: ft=yaml --- -libvirt: +values: libvirt_pkg: libvirt qemu_pkg: qemu python2_pkg: libvirt-python @@ -10,3 +10,4 @@ libvirt: libvirtd_config: /etc/libvirt/libvirtd.conf daemon_config_path: {} extra_pkgs: [] +... From e82d184a77d03725c5afcf0d5f73ba95a87875df Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 00:03:53 +0100 Subject: [PATCH 3/5] feat(map.jinja): split `osfamilymap.yaml` under `parameters/os_family/` --- libvirt/osfamilymap.yaml | 40 ------------------------ libvirt/parameters/os_family/Arch.yaml | 14 +++++++++ libvirt/parameters/os_family/Debian.yaml | 24 ++++++++++++++ libvirt/parameters/os_family/RedHat.yaml | 17 ++++++++++ libvirt/parameters/os_family/Suse.yaml | 20 ++++++++++++ 5 files changed, 75 insertions(+), 40 deletions(-) delete mode 100644 libvirt/osfamilymap.yaml create mode 100644 libvirt/parameters/os_family/Arch.yaml create mode 100644 libvirt/parameters/os_family/Debian.yaml create mode 100644 libvirt/parameters/os_family/RedHat.yaml create mode 100644 libvirt/parameters/os_family/Suse.yaml diff --git a/libvirt/osfamilymap.yaml b/libvirt/osfamilymap.yaml deleted file mode 100644 index 1770031..0000000 --- a/libvirt/osfamilymap.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['os_family'] based logic. -# You just need to add the key:values for an `os_family` that differ -# from `defaults.yaml`. -# Only add an `os_family` which is/will be supported by the formula -# -# If you do not need to provide defaults via the `os_family` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osfamilymap: {} ---- -Arch: - daemon_config_path: /etc/conf.d -Debian: - libvirt_pkg: libvirt-daemon-system - libvirt_service: libvirtd - qemu_pkg: qemu-kvm - python2_pkg: python-libvirt - python3_pkg: python3-libvirt - extra_pkgs: - - libguestfs0 - - libguestfs-tools - - gnutls-bin - - virt-top - daemon_config_path: /etc/default -RedHat: - qemu_pkg: qemu-kvm - extra_pkgs: - - libguestfs - daemon_config_path: /etc/sysconfig - -Suse: - libvirt_pkg: libvirt-daemon-qemu - qemu_pkg: qemu-kvm - extra_pkgs: - - libguestfs0 - python2_pkg: python2-libvirt-python - python3_pkg: python3-libvirt-python - daemon_config_path: /etc/sysconfig diff --git a/libvirt/parameters/os_family/Arch.yaml b/libvirt/parameters/os_family/Arch.yaml new file mode 100644 index 0000000..7d55435 --- /dev/null +++ b/libvirt/parameters/os_family/Arch.yaml @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Arch. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + daemon_config_path: /etc/conf.d +... diff --git a/libvirt/parameters/os_family/Debian.yaml b/libvirt/parameters/os_family/Debian.yaml new file mode 100644 index 0000000..b594d72 --- /dev/null +++ b/libvirt/parameters/os_family/Debian.yaml @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Debian. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + libvirt_pkg: libvirt-daemon-system + libvirt_service: libvirtd + qemu_pkg: qemu-kvm + python2_pkg: python-libvirt + python3_pkg: python3-libvirt + extra_pkgs: + - libguestfs0 + - libguestfs-tools + - gnutls-bin + - virt-top + daemon_config_path: /etc/default +... diff --git a/libvirt/parameters/os_family/RedHat.yaml b/libvirt/parameters/os_family/RedHat.yaml new file mode 100644 index 0000000..7b09659 --- /dev/null +++ b/libvirt/parameters/os_family/RedHat.yaml @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == RedHat. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + qemu_pkg: qemu-kvm + extra_pkgs: + - libguestfs + daemon_config_path: /etc/sysconfig +... diff --git a/libvirt/parameters/os_family/Suse.yaml b/libvirt/parameters/os_family/Suse.yaml new file mode 100644 index 0000000..bfc3fdb --- /dev/null +++ b/libvirt/parameters/os_family/Suse.yaml @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os_family') == Suse. +# You just need to add the key:values for this `os_family` that differ +# from `defaults.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os_family` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + libvirt_pkg: libvirt-daemon-qemu + qemu_pkg: qemu-kvm + extra_pkgs: + - libguestfs0 + python2_pkg: python2-libvirt-python + python3_pkg: python3-libvirt-python + daemon_config_path: /etc/sysconfig +... From 4255397e40466ef4782911989ab671bed160fcfa Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 00:04:02 +0100 Subject: [PATCH 4/5] feat(map.jinja): split `osmap.yaml` under `parameters/os/` --- libvirt/osmap.yaml | 19 ------------------- libvirt/parameters/os/CentOS.yaml | 15 +++++++++++++++ libvirt/parameters/os/Fedora.yaml | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 19 deletions(-) delete mode 100644 libvirt/osmap.yaml create mode 100644 libvirt/parameters/os/CentOS.yaml create mode 100644 libvirt/parameters/os/Fedora.yaml diff --git a/libvirt/osmap.yaml b/libvirt/osmap.yaml deleted file mode 100644 index e2d4ba4..0000000 --- a/libvirt/osmap.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['os'] based logic. -# You just need to add the key:values for an `os` that differ -# from `defaults.yaml` + `os_family.yaml`. -# Only add an `os` which is/will be supported by the formula -# -# If you do not need to provide defaults via the `os` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osmap: {} ---- -Fedora: - python2_pkg: python2-libvirt - python3_pkg: python3-libvirt - -CentOS: - python2_pkg: ~ - python3_pkg: python3-libvirt diff --git a/libvirt/parameters/os/CentOS.yaml b/libvirt/parameters/os/CentOS.yaml new file mode 100644 index 0000000..cf4e916 --- /dev/null +++ b/libvirt/parameters/os/CentOS.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os') == CentOS. +# You just need to add the key:values for this `os` that differ +# from `defaults.yaml` + `.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + python2_pkg: ~ + python3_pkg: python3-libvirt +... diff --git a/libvirt/parameters/os/Fedora.yaml b/libvirt/parameters/os/Fedora.yaml new file mode 100644 index 0000000..99df8a1 --- /dev/null +++ b/libvirt/parameters/os/Fedora.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('os') == Fedora. +# You just need to add the key:values for this `os` that differ +# from `defaults.yaml` + `.yaml` + `.yaml`. +# +# If you do not need to provide defaults via the `os` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + python2_pkg: python2-libvirt + python3_pkg: python3-libvirt +... From 365f71176a231d992426b79705d6c0ee5ede8c68 Mon Sep 17 00:00:00 2001 From: Daniel Dehennin Date: Wed, 25 Mar 2020 00:04:26 +0100 Subject: [PATCH 5/5] feat(map.jinja): split `osfingermap.yaml` under `parameters/osfinger/` --- libvirt/osfingermap.yaml | 22 ------------------- .../parameters/osfinger/CentOS Linux-7.yaml | 15 +++++++++++++ libvirt/parameters/osfinger/Ubuntu-16.04.yaml | 15 +++++++++++++ 3 files changed, 30 insertions(+), 22 deletions(-) delete mode 100644 libvirt/osfingermap.yaml create mode 100644 libvirt/parameters/osfinger/CentOS Linux-7.yaml create mode 100644 libvirt/parameters/osfinger/Ubuntu-16.04.yaml diff --git a/libvirt/osfingermap.yaml b/libvirt/osfingermap.yaml deleted file mode 100644 index 12bf431..0000000 --- a/libvirt/osfingermap.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# vim: ft=yaml -# -# Setup variables using grains['osfinger'] based logic. -# You just need to add the key:values for an `osfinger` that differ -# from `defaults.yaml` + `os_family.yaml` + `osmap.yaml`. -# Only add an `osfinger` which is/will be supported by the formula -# -# If you do not need to provide defaults via the `os_finger` grain, -# you will need to provide at least an empty dict in this file, e.g. -# osfingermap: {} ---- -## os: Ubuntu -# Xenial -Ubuntu-16.04: - libvirt_pkg: libvirt-bin - libvirt_service: libvirt-bin - -## os: CentOS -CentOS Linux-7: - python2_pkg: libvirt-python - python3_pkg: ~ diff --git a/libvirt/parameters/osfinger/CentOS Linux-7.yaml b/libvirt/parameters/osfinger/CentOS Linux-7.yaml new file mode 100644 index 0000000..89d508d --- /dev/null +++ b/libvirt/parameters/osfinger/CentOS Linux-7.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osfinger') == CentOS Linux-7. +# You just need to add the key:values for this `osfinger` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osfinger` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + python2_pkg: libvirt-python + python3_pkg: ~ +... diff --git a/libvirt/parameters/osfinger/Ubuntu-16.04.yaml b/libvirt/parameters/osfinger/Ubuntu-16.04.yaml new file mode 100644 index 0000000..0ea5750 --- /dev/null +++ b/libvirt/parameters/osfinger/Ubuntu-16.04.yaml @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +# +# Setup variables specific to salt['config.get']('osfinger') == Ubuntu-16.04. +# You just need to add the key:values for this `osfinger` that differ +# from `defaults.yaml`. +# +# If you do not need to provide defaults via the `osfinger` config, +# you can remove this file or provide at least an empty dict, e.g. +# values: {} +--- +values: + libvirt_pkg: libvirt-bin + libvirt_service: libvirt-bin +...