Merge pull request #5894 from ewolinetz/logging_defaults_from_existing

Automatic merge from submit-queue.

Update logging to use existing cluster deployment for defaults

This will allow us to use logging facts to set defaults of specific configurations such as ES index replicas and shard count.

The update to logging facts yields us output like: 
```json
"elasticsearch": {
            "clusterrolebindings": {}, 
            "configmaps": {
                "logging-elasticsearch": {
                    "elasticsearch.yml": {
                        "cloud": {
                            "kubernetes": {
                                "namespace": "${NAMESPACE}", 
                                "pod_label": "${POD_LABEL}", 
                                "pod_port": 9300
                            }
                        }, 
                        "cluster": {
                            "name": "${CLUSTER_NAME}"
                        }, 
                        "discovery": {
                            "type": "kubernetes", 
                            "zen.minimum_master_nodes": "${NODE_QUORUM}", 
                            "zen.ping.multicast.enabled": false
                        }, 
                        "gateway": {
                            "expected_nodes": "${RECOVER_EXPECTED_NODES}", 
                            "recover_after_nodes": "${NODE_QUORUM}", 
                            "recover_after_time": "${RECOVER_AFTER_TIME}"
                        }, 
                        "index": {
                            "number_of_replicas": 0, 
                            "number_of_shards": 1, 
                            "translog": {
                                "flush_threshold_period": "5m", 
                                "flush_threshold_size": "256mb"
                            }, 
                            "unassigned.node_left.delayed_timeout": "2m"
                        }, 
                        "io.fabric8.elasticsearch.authentication.users": [
                            "system.logging.kibana", 
                            "system.logging.fluentd", 
                            "system.logging.curator", 
                            "system.admin"
                        ], 
```

TODO:
- [x] Update logging facts to pull out settings from config maps
- [x] Move `openshift_sanitize_inventory/library/conditional_set_fact.py` up to repo level
- [x] Generate diffs against currently deployed configs and correctly patch in custom changes from customers
- [x] Use `conditional_set_fact` to easily set defaults for logging based on logging facts, or falling back to role defaults when not specified in the inventory
- [x] Update all components to follow patching configmaps

13 files changed, 312 insertions, 67 deletions

diff --git a/roles/openshift_sanitize_inventory/library/conditional_set_fact.py b/roles/lib_openshift/library/conditional_set_fact.py
index f61801714..363399f33 100644
--- a/roles/openshift_sanitize_inventory/library/conditional_set_fact.py
+++ b/roles/lib_openshift/library/conditional_set_fact.py
@@ -29,6 +29,10 @@ EXAMPLES = '''
     fact1: not_defined_variable
     fact2: defined_variable
 
+- name: Conditionally set fact falling back on default
+  conditional_set_fact:
+    fact1: not_defined_var | defined_variable
+
 '''
 
 
@@ -48,12 +52,14 @@ def run_module():
     is_changed = False
 
     for param in module.params['vars']:
-        other_var = module.params['vars'][param]
-
-        if other_var in module.params['facts']:
-            local_facts[param] = module.params['facts'][other_var]
-            if not is_changed:
-                is_changed = True
+        other_vars = module.params['vars'][param].replace(" ", "")
+
+        for other_var in other_vars.split('|'):
+            if other_var in module.params['facts']:
+                local_facts[param] = module.params['facts'][other_var]
+                if not is_changed:
+                    is_changed = True
+                break
 
     return module.exit_json(changed=is_changed,  # noqa: F405
                             ansible_facts=local_facts)
diff --git a/roles/openshift_logging/filter_plugins/openshift_logging.py b/roles/openshift_logging/filter_plugins/openshift_logging.py
index e1a5ea726..ba412b5a6 100644
--- a/roles/openshift_logging/filter_plugins/openshift_logging.py
+++ b/roles/openshift_logging/filter_plugins/openshift_logging.py
@@ -102,6 +102,28 @@ def serviceaccount_namespace(qualified_sa, default=None):
     return seg[-1]
 
 
+def flatten_dict(data, parent_key=None):
+    """ This filter plugin will flatten a dict and its sublists into a single dict
+    """
+    if not isinstance(data, dict):
+        raise RuntimeError("flatten_dict failed, expects to flatten a dict")
+
+    merged = dict()
+
+    for key in data:
+        if parent_key is not None:
+            insert_key = '.'.join((parent_key, key))
+        else:
+            insert_key = key
+
+        if isinstance(data[key], dict):
+            merged.update(flatten_dict(data[key], insert_key))
+        else:
+            merged[insert_key] = data[key]
+
+    return merged
+
+
 # pylint: disable=too-few-public-methods
 class FilterModule(object):
     ''' OpenShift Logging Filters '''
@@ -117,5 +139,6 @@ class FilterModule(object):
             'es_storage': es_storage,
             'serviceaccount_name': serviceaccount_name,
             'serviceaccount_namespace': serviceaccount_namespace,
-            'walk': walk
+            'walk': walk,
+            "flatten_dict": flatten_dict
         }
diff --git a/roles/openshift_logging/library/logging_patch.py b/roles/openshift_logging/library/logging_patch.py
new file mode 100644
index 000000000..d2c0bc456
--- /dev/null
+++ b/roles/openshift_logging/library/logging_patch.py
@@ -0,0 +1,112 @@
+#!/usr/bin/python
+
+""" Ansible module to help with creating context patch file with whitelisting for logging """
+
+import difflib
+import re
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+DOCUMENTATION = '''
+---
+module: logging_patch
+
+short_description: This will create a context patch file while giving ability
+  to whitelist some lines (excluding them from comparison)
+
+description:
+    - "To create configmap patches for logging"
+
+author:
+    - Eric Wolinetz ewolinet@redhat.com
+'''
+
+
+EXAMPLES = '''
+- logging_patch:
+    original_file: "{{ tempdir }}/current.yml"
+    new_file: "{{ configmap_new_file }}"
+    whitelist: "{{ configmap_protected_lines | default([]) }}"
+
+'''
+
+
+def account_for_whitelist(file_contents, white_list=None):
+    """ This method will remove lines that contain whitelist values from the content
+        of the file so that we aren't build a patch based on that line
+
+        Usage:
+
+          for file_contents:
+
+            index:
+              number_of_shards: {{ es_number_of_shards | default ('1') }}
+              number_of_replicas: {{ es_number_of_replicas | default ('0') }}
+              unassigned.node_left.delayed_timeout: 2m
+              translog:
+                flush_threshold_size: 256mb
+                flush_threshold_period: 5m
+
+
+          and white_list:
+
+            ['number_of_shards', 'number_of_replicas']
+
+
+        We would end up with:
+
+            index:
+              unassigned.node_left.delayed_timeout: 2m
+              translog:
+                flush_threshold_size: 256mb
+                flush_threshold_period: 5m
+
+    """
+
+    for line in white_list:
+        file_contents = re.sub(r".*%s:.*\n" % line, "", file_contents)
+
+    return file_contents
+
+
+def run_module():
+    """ The body of the module, we check if the variable name specified as the value
+        for the key is defined. If it is then we use that value as for the original key """
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            original_file=dict(type='str', required=True),
+            new_file=dict(type='str', required=True),
+            whitelist=dict(required=False, type='list', default=[])
+        ),
+        supports_check_mode=True
+    )
+
+    original_fh = open(module.params['original_file'], "r")
+    original_contents = original_fh.read()
+    original_fh.close()
+
+    original_contents = account_for_whitelist(original_contents, module.params['whitelist'])
+
+    new_fh = open(module.params['new_file'], "r")
+    new_contents = new_fh.read()
+    new_fh.close()
+
+    new_contents = account_for_whitelist(new_contents, module.params['whitelist'])
+
+    uni_diff = difflib.unified_diff(new_contents.splitlines(),
+                                    original_contents.splitlines(),
+                                    lineterm='')
+
+    return module.exit_json(changed=False,  # noqa: F405
+                            raw_patch="\n".join(uni_diff))
+
+
+def main():
+    """ main """
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/roles/openshift_logging/library/openshift_logging_facts.py b/roles/openshift_logging/library/openshift_logging_facts.py
index 98d0d1c4f..302a9b4c9 100644
--- a/roles/openshift_logging/library/openshift_logging_facts.py
+++ b/roles/openshift_logging/library/openshift_logging_facts.py
@@ -204,6 +204,14 @@ class OpenshiftLoggingFacts(OCBaseCommand):
             if comp is not None:
                 self.add_facts_for(comp, "services", name, dict())
 
+    # pylint: disable=too-many-arguments
+    def facts_from_configmap(self, comp, kind, name, config_key, yaml_file=None):
+        '''Extracts facts in logging namespace from configmap'''
+        if yaml_file is not None:
+            config_facts = yaml.load(yaml_file)
+            self.facts[comp][kind][name][config_key] = config_facts
+            self.facts[comp][kind][name]["raw"] = yaml_file
+
     def facts_for_configmaps(self, namespace):
         ''' Gathers facts for configmaps in logging namespace '''
         self.default_keys_for("configmaps")
@@ -214,7 +222,10 @@ class OpenshiftLoggingFacts(OCBaseCommand):
             name = item["metadata"]["name"]
             comp = self.comp(name)
             if comp is not None:
-                self.add_facts_for(comp, "configmaps", name, item["data"])
+                self.add_facts_for(comp, "configmaps", name, dict(item["data"]))
+                if comp in ["elasticsearch", "elasticsearch_ops"]:
+                    for config_key in item["data"]:
+                        self.facts_from_configmap(comp, "configmaps", name, config_key, item["data"][config_key])
 
     def facts_for_oauthclients(self, namespace):
         ''' Gathers facts for oauthclients used with logging '''
diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml
index 11f59652c..913478027 100644
--- a/roles/openshift_logging/tasks/install_logging.yaml
+++ b/roles/openshift_logging/tasks/install_logging.yaml
@@ -4,6 +4,9 @@
     oc_bin: "{{openshift_client_binary}}"
     openshift_logging_namespace: "{{openshift_logging_namespace}}"
 
+## This is include vs import because we need access to group/inventory variables
+- include_tasks: set_defaults_from_current.yml
+
 - name: Set logging project
   oc_project:
     state: present
diff --git a/roles/openshift_logging/tasks/patch_configmap_file.yaml b/roles/openshift_logging/tasks/patch_configmap_file.yaml
new file mode 100644
index 000000000..30087fe6a
--- /dev/null
+++ b/roles/openshift_logging/tasks/patch_configmap_file.yaml
@@ -0,0 +1,35 @@
+---
+## The purpose of this task file is to get a patch that is based on the diff
+##  between configmap_current_file and configmap_new_file. The module
+##  logging_patch takes the paths of two files to compare and also a list of
+##  variables whose line we exclude from the diffs.
+##  We then patch the new configmap file so that we can build a configmap
+##  using that file later. We then use oc apply to idempotenly modify any
+##  existing configmap.
+
+## The following variables are expected to be provided when including this task:
+# __configmap_output         -- This is provided to us from patch_configmap_files.yaml
+#                                it is a dict of the configmap where configmap_current_file exists
+# configmap_current_file     -- The name of the data file in the __configmap_output
+# configmap_new_file         -- The path to the file that we intend to oc apply later
+#                                we apply our generated patch to this file.
+# configmap_protected_lines  -- The list of variables to exclude from the diff
+
+- copy:
+    content: "{{ __configmap_output.results.results[0]['data'][configmap_current_file] }}"
+    dest: "{{ tempdir }}/current.yml"
+
+- logging_patch:
+    original_file: "{{ tempdir }}/current.yml"
+    new_file: "{{ configmap_new_file }}"
+    whitelist: "{{ configmap_protected_lines | default([]) }}"
+  register: patch_output
+
+- copy:
+    content: "{{ patch_output.raw_patch }}\n"
+    dest: "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0
+
+- command: >
+    patch --force --quiet -u "{{ configmap_new_file }}" "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0
diff --git a/roles/openshift_logging/tasks/patch_configmap_files.yaml b/roles/openshift_logging/tasks/patch_configmap_files.yaml
new file mode 100644
index 000000000..74a9cc287
--- /dev/null
+++ b/roles/openshift_logging/tasks/patch_configmap_files.yaml
@@ -0,0 +1,31 @@
+---
+## The purpose of this task file is to take in a list of configmap files provided
+##  in the variable configmap_file_names, which correspond to the data sections
+##  within a configmap. We iterate over each of these files and create a patch
+##  from the diff between current_file and new_file to try to maintain any custom
+##  changes that a user may have made to a currently deployed configmap while
+##  trying to idempotently update with any role provided files.
+
+## The following variables are expected to be provided when including this task:
+# configmap_name        -- This is the name of the configmap that the files exist in
+# configmap_namespace   -- The namespace that the configmap lives in
+# configmap_file_names  -- This is expected to be passed in as a dict
+#   current_file        -- The name of the data entry within the configmap
+#   new_file            -- The file path to the file we are comparing to current_file
+#   protected_lines     -- List of variables whose line will be excluded when creating a diff
+
+- oc_configmap:
+    name: "{{ configmap_name }}"
+    state: list
+    namespace: "{{ configmap_namespace }}"
+  register: __configmap_output
+
+- when: __configmap_output.results.stderr is undefined
+  include_tasks: patch_configmap_file.yaml
+  vars:
+    configmap_current_file: "{{ configmap_files.current_file }}"
+    configmap_new_file: "{{ configmap_files.new_file }}"
+    configmap_protected_lines: "{{ configmap_files.protected_lines | default([]) }}"
+  with_items: "{{ configmap_file_names }}"
+  loop_control:
+    loop_var: configmap_files
diff --git a/roles/openshift_logging/tasks/set_defaults_from_current.yml b/roles/openshift_logging/tasks/set_defaults_from_current.yml
new file mode 100644
index 000000000..dde362abe
--- /dev/null
+++ b/roles/openshift_logging/tasks/set_defaults_from_current.yml
@@ -0,0 +1,34 @@
+---
+
+## We are pulling default values from configmaps if they exist already
+## Using conditional_set_fact allows us to set the value of a variable based on
+##  the value of another one, if it is already defined. Else we don't set the
+##  left hand side (it stays undefined as well).
+
+## conditional_set_fact allows us to specify a fact source, so first we try to
+##  set variables in the logging-elasticsearch & logging-elasticsearch-ops configmaps
+##  afterwards we set the value of the variable based on the value in the inventory
+##  but fall back to using the value from a configmap as a default. If neither is set
+##  then the variable remains undefined and the role default will be used.
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_number_of_shards: index.number_of_shards
+      __openshift_logging_es_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch'] is defined
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_ops_number_of_shards: index.number_of_shards
+      __openshift_logging_es_ops_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops'] is defined
+
+- conditional_set_fact:
+    facts: "{{ hostvars[inventory_hostname] }}"
+    vars:
+      openshift_logging_es_number_of_shards: openshift_logging_es_number_of_shards | __openshift_logging_es_number_of_shards
+      openshift_logging_es_number_of_replicas: openshift_logging_es_number_of_replicas | __openshift_logging_es_number_of_replicas
+      openshift_logging_es_ops_number_of_shards: openshift_logging_es_ops_number_of_shards | __openshift_logging_es_ops_number_of_shards
+      openshift_logging_es_ops_number_of_replicas: openshift_logging_es_ops_number_of_replicas | __openshift_logging_es_ops_number_of_replicas
diff --git a/roles/openshift_logging_curator/tasks/main.yaml b/roles/openshift_logging_curator/tasks/main.yaml
index 524e239b7..53b464113 100644
--- a/roles/openshift_logging_curator/tasks/main.yaml
+++ b/roles/openshift_logging_curator/tasks/main.yaml
@@ -54,14 +54,17 @@
 - copy:
     src: curator.yml
     dest: "{{ tempdir }}/curator.yml"
-  when: curator_config_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{ curator_config_contents }}"
-    dest: "{{ tempdir }}/curator.yml"
-  when: curator_config_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-curator"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "config.yaml"
+        new_file: "{{ tempdir }}/curator.yml"
 
 - name: Set Curator configmap
   oc_configmap:
diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml
index 6ddeb122e..9e7646379 100644
--- a/roles/openshift_logging_elasticsearch/tasks/main.yaml
+++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml
@@ -168,33 +168,31 @@
   when: es_logging_contents is undefined
   changed_when: no
 
-- set_fact:
-    __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}"
-    __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}"
-
 - template:
     src: elasticsearch.yml.j2
     dest: "{{ tempdir }}/elasticsearch.yml"
   vars:
     allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}"
-    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}"
-    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}"
+    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}"
+    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}"
     es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}"
 
   when: es_config_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{ es_logging_contents }}"
-    dest: "{{ tempdir }}/elasticsearch-logging.yml"
-  when: es_logging_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{ es_config_contents }}"
-    dest: "{{ tempdir }}/elasticsearch.yml"
-  when: es_config_contents is defined
-  changed_when: no
+# create diff between current configmap files and our current files
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-elasticsearch"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "elasticsearch.yml"
+        new_file: "{{ tempdir }}/elasticsearch.yml"
+        protected_lines: ["number_of_shards", "number_of_replicas"]
+      - current_file: "logging.yml"
+        new_file: "{{ tempdir }}/elasticsearch-logging.yml"
 
 - name: Set ES configmap
   oc_configmap:
diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml
index 08d7561ac..486cfb8bc 100644
--- a/roles/openshift_logging_fluentd/tasks/main.yaml
+++ b/roles/openshift_logging_fluentd/tasks/main.yaml
@@ -108,38 +108,28 @@
     dest: "{{ tempdir }}/fluent.conf"
   vars:
     deploy_type: "{{ openshift_logging_fluentd_deployment_type }}"
-  when: fluentd_config_contents is undefined
-  changed_when: no
 
 - copy:
     src: fluentd-throttle-config.yaml
     dest: "{{ tempdir }}/fluentd-throttle-config.yaml"
-  when: fluentd_throttle_contents is undefined
-  changed_when: no
 
 - copy:
     src: secure-forward.conf
     dest: "{{ tempdir }}/secure-forward.conf"
-  when: fluentd_secureforward_contents is undefined
-  changed_when: no
-
-- copy:
-    content: "{{ fluentd_config_contents }}"
-    dest: "{{ tempdir }}/fluent.conf"
-  when: fluentd_config_contents is defined
-  changed_when: no
 
-- copy:
-    content: "{{ fluentd_throttle_contents }}"
-    dest: "{{ tempdir }}/fluentd-throttle-config.yaml"
-  when: fluentd_throttle_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{ fluentd_secureforward_contents }}"
-    dest: "{{ tempdir }}/secure-forward.conf"
-  when: fluentd_secureforward_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-fluentd"
+    configmap_namespace: "logging"
+    configmap_file_names:
+      - current_file: "fluent.conf"
+        new_file: "{{ tempdir }}/fluent.conf"
+      - current_file: "throttle-config.yaml"
+        new_file: "{{ tempdir }}/fluentd-throttle-config.yaml"
+      - current_file: "secure-forward.conf"
+        new_file: "{{ tempdir }}/secure-forward.conf"
 
 - name: Set Fluentd configmap
   oc_configmap:
diff --git a/roles/openshift_logging_mux/tasks/main.yaml b/roles/openshift_logging_mux/tasks/main.yaml
index 59a6301d7..a281c6a53 100644
--- a/roles/openshift_logging_mux/tasks/main.yaml
+++ b/roles/openshift_logging_mux/tasks/main.yaml
@@ -88,26 +88,24 @@
 - copy:
     src: fluent.conf
     dest: "{{mktemp.stdout}}/fluent-mux.conf"
-  when: fluentd_mux_config_contents is undefined
   changed_when: no
 
 - copy:
     src: secure-forward.conf
     dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
-  when: fluentd_mux_securefoward_contents is undefined
   changed_when: no
 
-- copy:
-    content: "{{fluentd_mux_config_contents}}"
-    dest: "{{mktemp.stdout}}/fluent-mux.conf"
-  when: fluentd_mux_config_contents is defined
-  changed_when: no
-
-- copy:
-    content: "{{fluentd_mux_secureforward_contents}}"
-    dest: "{{mktemp.stdout}}/secure-forward-mux.conf"
-  when: fluentd_mux_secureforward_contents is defined
-  changed_when: no
+- include_role:
+    name: openshift_logging
+    tasks_from: patch_configmap_files.yaml
+  vars:
+    configmap_name: "logging-mux"
+    configmap_namespace: "{{ openshift_logging_mux_namespace }}"
+    configmap_file_names:
+      - current_file: "fluent.conf"
+        new_file: "{{ tempdir }}/fluent-mux.conf"
+      - current_file: "secure-forward.conf"
+        new_file: "{{ tempdir }}/secure-forward-mux.conf"
 
 - name: Set Mux configmap
   oc_configmap:
diff --git a/roles/openshift_sanitize_inventory/meta/main.yml b/roles/openshift_sanitize_inventory/meta/main.yml
index 324ba06d8..cde3eccb6 100644
--- a/roles/openshift_sanitize_inventory/meta/main.yml
+++ b/roles/openshift_sanitize_inventory/meta/main.yml
@@ -14,3 +14,4 @@ galaxy_info:
   - system
 dependencies:
 - role: lib_utils
+- role: lib_openshift