feat(map.jinja): load configuration values from configurables sources

libvirt/map.jinja

@@ -3,42 +3,118 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{#- Start imports as #}
-{%- import_yaml tplroot ~ "/defaults.yaml" as default_settings %}
-{%- import_yaml tplroot ~ "/osfamilymap.yaml" as osfamilymap %}
-{%- import_yaml tplroot ~ "/osmap.yaml" as osmap %}
-{%- import_yaml tplroot ~ "/osfingermap.yaml" as osfingermap %}
-
-{#- Retrieve the config dict only once #}
-{%- set _config = salt['config.get'](tplroot, default={}) %}
-
-{%- set defaults = salt['grains.filter_by'](
-      default_settings,
-      default=tplroot,
-      merge=salt['grains.filter_by'](
-        osfamilymap,
-        grain='os_family',
-        merge=salt['grains.filter_by'](
-          osmap,
-          grain='os',
-          merge=salt['grains.filter_by'](
-            osfingermap,
-            grain='osfinger',
-            merge=salt['grains.filter_by'](
-              _config,
-              default='lookup'
-            )
-          )
-        )
-      )
-    )
- %}
-
-{%- set config = salt['grains.filter_by'](
-      {'defaults': defaults},
-      default='defaults',
-      merge=_config
-    )
-%}
-
-{%- set libvirt_settings = config %}
+
+{#- Where to lookup parameters source files #}
+{%- set map_sources_dir = tplroot | path_join('parameters') %}
+
+{#- Load defaults first to allow per formula default map.jinja configuration #}
+{%- set _defaults_filename = map_sources_dir | path_join('defaults.yaml') %}
+{%- do salt['log.debug']('map.jinja: initialise parameters from ' ~ _defaults_filename ) %}
+{%- import_yaml _defaults_filename as default_settings %}
+
+{#- List of sources to lookup for parameters #}
+{%- do salt['log.debug']("map.jinja: lookup 'map_jinja' configuration sources") %}
+{#- Fallback to previously used grains plus minion `id` #}
+{%- set map_sources = ['osarch', 'os_family', 'os', 'osfinger', 'config_get_lookup', 'config_get', 'id'] %}
+{#- Configure map.jinja from defaults.yaml #}
+{%- set map_sources = default_settings | traverse('values:map_jinja:sources', map_sources) %}
+
+{#- Lookup global sources #}
+{%- set map_sources = salt['config.get']('map_jinja:sources', map_sources) %}
+{#- Lookup per formula sources #}
+{%- set map_sources = salt['config.get'](tplroot ~ ':map_jinja:sources', map_sources) %}
+
+{%- do salt['log.debug']('map.jinja: load parameters with sources from ' ~ map_sources) %}
+
+
+{#- Work around assignment inside for loop #}
+{#- load configuration values used in `config.get` merging strategies #}
+{%- set _config = {'stack': default_settings.get('values', {}),
+                   'merge_strategy': salt['config.get'](tplroot ~ ':strategy', None),
+                   'merge_lists': salt['config.get'](tplroot ~ ':merge_lists', False),
+                   'cli': salt['config.get']('__cli'),
+                   'root_dir': salt['config.get']('root_dir')
+                  } %}
+
+{#- `config.get` merge option works only for `minion` or local `salt-call` #}
+{%- if _config['cli'] == 'salt-minion'
+       or (_config['cli'] == 'salt-call' and not _config['root_dir'].endswith('/running_data')) %}
+  {%- do _config.update({'merge_opt': {'merge': _config['merge_strategy']},
+                         'merge_msg': ", merge: strategy='" ~ _config['merge_strategy'] ~ "'"}) %}
+{%- else %}
+  {#- no `config.get` merge option for `salt-ssh` or `unknown` cli #}
+  {%- if _config['merge_strategy'] %}
+    {%- do salt['log.error']("map.jinja: the 'merge' option of 'config.get' is skipped with salt cli '"
+                             ~ _config['cli'] | replace('call', 'ssh') | default('unknown', True) ~ "'")
+      %}
+  {%- endif %}
+  {%- do _config.update({'merge_opt': {},
+                         'merge_msg': ''}) %}
+{%- endif %}
+
+
+{#- process each `map.jinja` source #}
+{%- for map_source in map_sources %}
+  {%- if map_source  in ['config_get', 'config_get_lookup'] %}
+    {%- set _config_key = {'config_get': tplroot,
+                           'config_get_lookup': tplroot ~ ':lookup'}.get(map_source) %}
+    {%- set _config_type = {'config_get': 'configuration',
+                            'config_get_lookup': 'lookup'}.get(map_source) %}
+
+    {%- do salt['log.debug']("map.jinja: retrieve formula " ~ _config_type
+                             ~ " with 'config.get'"
+                             ~ _config['merge_msg']
+                             ) %}
+    {%- set _config_get = salt['config.get'](_config_key, default={}, **_config['merge_opt']) %}
+
+    {#- `slsutil.merge` defaults to `smart` instead of `None` for `config.get` #}
+    {%- set _strategy = _config['merge_strategy'] | default('smart', boolean=True) %}
+    {%- do salt['log.debug']("map.jinja: merge formula " ~ _config_type
+                             ~ " retrieved with 'config.get'"
+                             ~ ", merge: strategy='" ~ _strategy
+                             ~ "', lists='" ~ _config['merge_lists'] ~ "'"
+                            ) %}
+    {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'],
+                                                          _config_get,
+                                                          strategy=_strategy,
+                                                          merge_lists=_config['merge_lists'])})
+      %}
+  {%- else %}
+    {#- Lookup the grain/pillar/... #}
+    {#- Fallback to use the source name as a direct filename #}
+    {%- set map_values = salt['config.get'](map_source, []) %}
+
+    {#- Mangle `map_source` to use it as literal path #}
+    {%- if map_values|length == 0 %}
+      {%- set map_source_parts = map_source.split('/') %}
+      {%- set map_source = map_source_parts[0:-1] | join('/') %}
+      {%- set map_values = map_source_parts[-1].rstrip('.yaml') %}
+    {%- endif %}
+
+    {#- Some configuration return list #}
+    {%- if map_values is string %}
+      {%- set map_values = [map_values] %}
+    {%- endif %}
+
+    {%- for map_value in map_values %}
+      {%- set yamlfile = map_sources_dir | path_join(map_source, map_value ~ '.yaml') %}
+      {%- do salt['log.debug']('map.jinja: load parameters from file ' ~ yamlfile) %}
+      {%- load_yaml as loaded_values %}
+        {%- include yamlfile ignore missing %}
+      {%- endload %}
+
+      {%- if loaded_values %}
+        {#- Merge loaded values on the stack #}
+        {%- do salt['log.debug']('map.jinja: merge parameters from ' ~ yamlfile) %}
+        {%- do _config.update({'stack': salt['slsutil.merge'](_config['stack'],
+                                                              loaded_values.get('values', {}),
+                                                              strategy=loaded_values.get('strategy', 'smart'),
+                                                              merge_lists=loaded_values.get('merge_lists', False) | to_bool)})
+          %}
+      {%- endif %}
+    {%- endfor %}
+  {%- endif %}
+{%- endfor %}
+
+{%- do salt['log.debug']("map.jinja: save parameters in variable 'libvirt_settings'") %}
+{%- set libvirt_settings = _config['stack'] %}

libvirt/defaults.yaml → libvirt/parameters/defaults.yaml

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # vim: ft=yaml
 ---
-libvirt:
+values:
   libvirt_pkg: libvirt
   qemu_pkg: qemu
   python2_pkg: libvirt-python
@@ -10,3 +10,4 @@ libvirt:
   libvirtd_config: /etc/libvirt/libvirtd.conf
   daemon_config_path: {}
   extra_pkgs: []
+...

libvirt/parameters/os/CentOS.yaml

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os') == CentOS.
+# You just need to add the key:values for this `os` that differ
+# from `defaults.yaml` + `<osarch>.yaml` + `<os_family>.yaml`.
+#
+# If you do not need to provide defaults via the `os` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  python2_pkg: ~
+  python3_pkg: python3-libvirt
+...

libvirt/parameters/os/Fedora.yaml

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os') == Fedora.
+# You just need to add the key:values for this `os` that differ
+# from `defaults.yaml` + `<osarch>.yaml` + `<os_family>.yaml`.
+#
+# If you do not need to provide defaults via the `os` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  python2_pkg: python2-libvirt
+  python3_pkg: python3-libvirt
+...

libvirt/parameters/os_family/Arch.yaml

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Arch.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  daemon_config_path: /etc/conf.d
+...

libvirt/parameters/os_family/Debian.yaml

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Debian.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  libvirt_pkg: libvirt-daemon-system
+  libvirt_service: libvirtd
+  qemu_pkg: qemu-kvm
+  python2_pkg: python-libvirt
+  python3_pkg: python3-libvirt
+  extra_pkgs:
+    - libguestfs0
+    - libguestfs-tools
+    - gnutls-bin
+    - virt-top
+  daemon_config_path: /etc/default
+...

libvirt/parameters/os_family/RedHat.yaml

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == RedHat.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  qemu_pkg: qemu-kvm
+  extra_pkgs:
+    - libguestfs
+  daemon_config_path: /etc/sysconfig
+...

libvirt/parameters/os_family/Suse.yaml

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('os_family') == Suse.
+# You just need to add the key:values for this `os_family` that differ
+# from `defaults.yaml` + `<osarch>.yaml`.
+#
+# If you do not need to provide defaults via the `os_family` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  libvirt_pkg: libvirt-daemon-qemu
+  qemu_pkg: qemu-kvm
+  extra_pkgs:
+    - libguestfs0
+  python2_pkg: python2-libvirt-python
+  python3_pkg: python3-libvirt-python
+  daemon_config_path: /etc/sysconfig
+...

libvirt/parameters/osfinger/CentOS Linux-7.yaml

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('osfinger') == CentOS Linux-7.
+# You just need to add the key:values for this `osfinger` that differ
+# from `defaults.yaml`.
+#
+# If you do not need to provide defaults via the `osfinger` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  python2_pkg: libvirt-python
+  python3_pkg: ~
+...

libvirt/parameters/osfinger/Ubuntu-16.04.yaml

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
+#
+# Setup variables specific to salt['config.get']('osfinger') == Ubuntu-16.04.
+# You just need to add the key:values for this `osfinger` that differ
+# from `defaults.yaml`.
+#
+# If you do not need to provide defaults via the `osfinger` config,
+# you can remove this file or provide at least an empty dict, e.g.
+# values: {}
+---
+values:
+  libvirt_pkg: libvirt-bin
+  libvirt_service: libvirt-bin
+...