diff --git a/TEMPLATE/parameters/os/Fedora.yaml b/TEMPLATE/parameters/os/Fedora.yaml index 52f393e3..ebbe2e46 100644 --- a/TEMPLATE/parameters/os/Fedora.yaml +++ b/TEMPLATE/parameters/os/Fedora.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/os/Ubuntu.yaml b/TEMPLATE/parameters/os/Ubuntu.yaml index 4dc91395..324cb3d3 100644 --- a/TEMPLATE/parameters/os/Ubuntu.yaml +++ b/TEMPLATE/parameters/os/Ubuntu.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/os_family/Arch.yaml b/TEMPLATE/parameters/os_family/Arch.yaml index 5f3f78b7..266641fe 100644 --- a/TEMPLATE/parameters/os_family/Arch.yaml +++ b/TEMPLATE/parameters/os_family/Arch.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/os_family/Debian.yaml b/TEMPLATE/parameters/os_family/Debian.yaml index e7c6ce34..c3a30541 100644 --- a/TEMPLATE/parameters/os_family/Debian.yaml +++ b/TEMPLATE/parameters/os_family/Debian.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/os_family/FreeBSD.yaml b/TEMPLATE/parameters/os_family/FreeBSD.yaml index c5d3a048..1514cd0d 100644 --- a/TEMPLATE/parameters/os_family/FreeBSD.yaml +++ b/TEMPLATE/parameters/os_family/FreeBSD.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: rootgroup: wheel diff --git a/TEMPLATE/parameters/os_family/OpenBSD.yaml b/TEMPLATE/parameters/os_family/OpenBSD.yaml index 5a26d4bc..4feba1d9 100644 --- a/TEMPLATE/parameters/os_family/OpenBSD.yaml +++ b/TEMPLATE/parameters/os_family/OpenBSD.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: rootgroup: wheel diff --git a/TEMPLATE/parameters/os_family/RedHat.yaml b/TEMPLATE/parameters/os_family/RedHat.yaml index c1e701ef..9af605e9 100644 --- a/TEMPLATE/parameters/os_family/RedHat.yaml +++ b/TEMPLATE/parameters/os_family/RedHat.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/os_family/Suse.yaml b/TEMPLATE/parameters/os_family/Suse.yaml index c828ec5d..07bb2416 100644 --- a/TEMPLATE/parameters/os_family/Suse.yaml +++ b/TEMPLATE/parameters/os_family/Suse.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `os_family` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/osarch/386.yaml b/TEMPLATE/parameters/osarch/386.yaml index e6d4154f..377787a7 100644 --- a/TEMPLATE/parameters/osarch/386.yaml +++ b/TEMPLATE/parameters/osarch/386.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: 386 diff --git a/TEMPLATE/parameters/osarch/amd64.yaml b/TEMPLATE/parameters/osarch/amd64.yaml index 54fc715d..5eee548e 100644 --- a/TEMPLATE/parameters/osarch/amd64.yaml +++ b/TEMPLATE/parameters/osarch/amd64.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: amd64 diff --git a/TEMPLATE/parameters/osarch/arm64.yaml b/TEMPLATE/parameters/osarch/arm64.yaml index 5c5beb8e..8846d066 100644 --- a/TEMPLATE/parameters/osarch/arm64.yaml +++ b/TEMPLATE/parameters/osarch/arm64.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: arm64 diff --git a/TEMPLATE/parameters/osarch/armv6l.yaml b/TEMPLATE/parameters/osarch/armv6l.yaml index f3d53011..c59440b4 100644 --- a/TEMPLATE/parameters/osarch/armv6l.yaml +++ b/TEMPLATE/parameters/osarch/armv6l.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: armv6l diff --git a/TEMPLATE/parameters/osarch/armv7l.yaml b/TEMPLATE/parameters/osarch/armv7l.yaml index 7c7f9518..b949603d 100644 --- a/TEMPLATE/parameters/osarch/armv7l.yaml +++ b/TEMPLATE/parameters/osarch/armv7l.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: armv7l diff --git a/TEMPLATE/parameters/osarch/ppc64le.yaml b/TEMPLATE/parameters/osarch/ppc64le.yaml index 5fc1d7d9..eab2c457 100644 --- a/TEMPLATE/parameters/osarch/ppc64le.yaml +++ b/TEMPLATE/parameters/osarch/ppc64le.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: ppc64le diff --git a/TEMPLATE/parameters/osarch/s390x.yaml b/TEMPLATE/parameters/osarch/s390x.yaml index ab24eca6..c8651c6d 100644 --- a/TEMPLATE/parameters/osarch/s390x.yaml +++ b/TEMPLATE/parameters/osarch/s390x.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: s390x diff --git a/TEMPLATE/parameters/osarch/x86_64.yaml b/TEMPLATE/parameters/osarch/x86_64.yaml index 77d1bd6e..2c009dd1 100644 --- a/TEMPLATE/parameters/osarch/x86_64.yaml +++ b/TEMPLATE/parameters/osarch/x86_64.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osarch` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: arch: amd64 diff --git a/TEMPLATE/parameters/osfinger/CentOS-6.yaml b/TEMPLATE/parameters/osfinger/CentOS-6.yaml index 3fb4afbb..714e3ede 100644 --- a/TEMPLATE/parameters/osfinger/CentOS-6.yaml +++ b/TEMPLATE/parameters/osfinger/CentOS-6.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osfinger` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: pkg: diff --git a/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml b/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml index c16dbda9..67048dd2 100644 --- a/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml +++ b/TEMPLATE/parameters/osfinger/Ubuntu-18.04.yaml @@ -8,16 +8,6 @@ # If you do not need to provide defaults via the `osfinger` config, # you can remove this file or provide at least an empty dict, e.g. # values: {} -# -# This file will be merged with `salt.slsutil.merge`: -# -# You can select the merging strategy by defining the `strategy` dict -# with one of: `aggregate`, `list`, `overwrite`, `recurse`, `smart`, e.g. -# strategy: 'recurse' -# -# If you use the `recurse` or `overwrite` strategy, you can aggregate -# the lists by defining the `merge_lists` dict with a boolean, e.g. -# merge_lists: 'true' --- values: config: /etc/TEMPLATE.d/custom-ubuntu-18.04.conf diff --git a/docs/README.rst b/docs/README.rst index 7e252f8e..fa649cb7 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -33,7 +33,11 @@ which contains the currently released version. This formula is versioned accordi See `Formula Versioning Section `_ for more details. -If you need (non-default) configuration, please pay attention to the ``pillar.example`` file and/or `Special notes`_ section. +If you need (non-default) configuration, please pay attention: + +- to the ``pillar.example`` file +- to `how to configure formula with map.jinja `_ +- and/or `Special notes`_ section Contributing to this repo ------------------------- diff --git a/docs/map.jinja.rst b/docs/map.jinja.rst new file mode 100644 index 00000000..01d53a42 --- /dev/null +++ b/docs/map.jinja.rst @@ -0,0 +1,164 @@ +.. _map.jinja: + +`map.jinja`: gather formula configuration values +================================================ + +The `documentation `_ gives a quick view of the use of a `map.jinja` to gather parameters values for a formula. + +The today best practice is to let `map.jinja` handle parameters from all sources and avoid the call of pillars, grains or configuration from `sls` files and template directly. + + +.. contents:: **Table of Contents** + + +How to set configuration values of a formula +-------------------------------------------- + +The `map.jinja` use several sources where to lookup parameter values: + +- YAML files located in the `fileserver `_ +- configuration collected by `salt['config.get'] `_ + +The `pillars `_ are rendered by the SaltStack master and could became costly with lots of minions with many pillar values. + +As a good practice, you should: + +- store non secret data in YAML files distributed by the `fileserver `_ +- store sensible data + - in pillar (and look for the use of something like `pillar.vault `_) + - in `SDB `_ (and look for the use of something like `sbd.vault `_) + + +Configuration from YAML files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Each formula comes with one or more YAML files, all located under the `/parameters` directory. The parameter values are initialised with the mandatory `defaults.yaml`. It should contain sane default values for the formula. + +Then, `map.jinja` will load configuration using `salt['config.get'] `_ from a configurable list, by default, this list is the following: + +- `osarch`: the CPU architecture of the minion +- `os_family`: the family of the operating system (e.g. `Debian` for an `Ubuntu`) +- `os`: the name of the operating system (e.g. `Ubuntu`) +- `osfinger`: the concatenation of the operating system name and it's version string (e.g. `Debian-10`) +- `id`: the `ID` of the minion + +After the configuration values are initialised with `defaults.yaml`, for each configuration of the list, in the order, `map.jinja` will: + +- lookup the value of the configuration +- load the values of `parameters//.yaml` +- merge the loaded values with the previous ones using `salt.slsutil.merge `_ + +Each YAML parameter file: + +- is optional, there will be no error if the file does not exists +- when it exists + - the configuration values must be under the top level `values` key + - the merging strategy of `salt.slsutil.merge` can be configured by the top level `strategy` key, for example `strategy: 'recurse'`, the default is `smart` + - the merging of list for the `recurse` and `overwrite` merging strategy can be configured with the top level key `merge_lists`, for example `merge_lists: 'true'` + +If the `config.get` lookup failed, the configuration name will be used literally as a custom path to a YAML file, for example: `any/path/can/be/used/here.yaml` will result in the loading of `parameters/any/path/can/be/used/here.yaml`. + +You can override the list of configuration to lookup by setting `map_jinja:sources` from 3 places: + +- `defaults.yaml` for the author of the formula +- pillar root (or anywhere reachable by `salt['config.get']` +- under `:map_jinja:sources` + +Configuration from `salt['config.get']` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +After the configuration is loaded from YAML files, `map.jinja` lookup the `` with `salt['config.get'] `_ and then merge with the previously loaded values: + +- first the `:lookup` dict +- then the complete `` dict + +Global view of the order of preferences +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To make things clear, here is a complete example of the load order of formula configuration values for an `AMD64` `Ubuntu 18.04` minion named `minion1.example.net`: + +#. `parameters/defaults.yaml` +#. `parameters/osarch/amd64.yaml` +#. `parameters/os_family/Debian.yaml` +#. `parameters/os/Ubunta.yaml` +#. `parameters/osfinger/Ubunta-18.04.yaml` +#. `parameters/id/minion1.example.net` +#. `salt['config.get'](':lookup')` +#. `salt['config.get']('')` + + +Use formula configuration values in `sls` +----------------------------------------- + +The good practice for `map.jinja` are: + +- to be located at the root of the formula named directory (e.g. `mysql-formula/mysql/map.jinja`) +- to export a variable with the same name than the formula itself. As an example, for `mysql-formula` it will be `mysql`. + +Here is the best way to use it in an `sls` file: + +.. code-block:: sls + + {#- Get the `tplroot` from `tpldir` #} + {%- set tplroot = tpldir.split('/')[0] %} + {%- from tplroot | path_join('map.jinja') import TEMPLATE with context %} + + test-does-nothing-but-display-TEMPLATE-as-json: + test.nop: + - name: {{ TEMPLATE | json }} + + + +Use formula configuration values in templates +--------------------------------------------- + +When you need to process salt templates, you should avoid calling `salt['config.get']` (or `salt['pillar.get']` and `salt['grains.get']`) directly from the template. All the needed values should be available within the variable exported by `map.jinja`. + +Here is an example based on `template-formula/TEMPLATE/config/file.sls` + +.. code-block:: sls + + # -*- coding: utf-8 -*- + # vim: ft=sls + + {#- Get the `tplroot` from `tpldir` #} + {%- set tplroot = tpldir.split('/')[0] %} + {%- set sls_package_install = tplroot ~ '.package.install' %} + {%- from tplroot ~ "/map.jinja" import TEMPLATE with context %} + {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} + + include: + - {{ sls_package_install }} + + TEMPLATE-config-file-file-managed: + file.managed: + - name: {{ TEMPLATE.config }} + - source: {{ files_switch(['example.tmpl'], + lookup='TEMPLATE-config-file-file-managed' + ) + }} + - mode: 644 + - user: root + - group: {{ TEMPLATE.rootgroup }} + - makedirs: True + - template: jinja + - require: + - sls: {{ sls_package_install }} + - context: + TEMPLATE: {{ TEMPLATE | json }} + +This `sls` file expose a `TEMPLATE` context variable to the jinja template which could be used like this: + +.. code-block:: jinja + + ######################################################################## + # File managed by Salt at <{{ source }}>. + # Your changes will be overwritten. + ######################################################################## + + This is another example file from SaltStack template-formula. + + # This is here for testing purposes + {{ TEMPLATE | json }} + + winner of the merge: {{ TEMPLATE['winner'] }}