Merge pull request #5766 from jcantrill/1489498_preserve_replica_count

bug 1489498. preserve replica and shard settings

6 files changed, 63 insertions, 6 deletions

diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml
index 6e7e2557f..6aff89dbf 100644
--- a/roles/openshift_logging/defaults/main.yml
+++ b/roles/openshift_logging/defaults/main.yml
@@ -98,8 +98,6 @@ openshift_logging_es_storage_group: "65534"
 openshift_logging_es_nodeselector: {}
 # openshift_logging_es_config is a hash to be merged into the defaults for the elasticsearch.yaml
 openshift_logging_es_config: {}
-openshift_logging_es_number_of_shards: 1
-openshift_logging_es_number_of_replicas: 0
 
 # for exposing es to external (outside of the cluster) clients
 openshift_logging_es_allow_external: False
diff --git a/roles/openshift_logging/filter_plugins/openshift_logging.py b/roles/openshift_logging/filter_plugins/openshift_logging.py
index 330e7e59a..959573635 100644
--- a/roles/openshift_logging/filter_plugins/openshift_logging.py
+++ b/roles/openshift_logging/filter_plugins/openshift_logging.py
@@ -17,6 +17,22 @@ def es_storage(os_logging_facts, dc_name, pvc_claim, root='elasticsearch'):
     return dict(kind='emptydir')
 
 
+def walk(source, path, default, delimiter='.'):
+    '''Walk the sourch hash given the path and return the value or default if not found'''
+    if not isinstance(source, dict):
+        raise RuntimeError('The source is not a walkable dict: {} path: {}'.format(source, path))
+    keys = path.split(delimiter)
+    max_depth = len(keys)
+    cur_depth = 0
+    while cur_depth < max_depth:
+        if keys[cur_depth] in source:
+            source = source[keys[cur_depth]]
+            cur_depth = cur_depth + 1
+        else:
+            return default
+    return source
+
+
 def random_word(source_alpha, length):
     ''' Returns a random word given the source of characters to pick from and resulting length '''
     return ''.join(random.choice(source_alpha) for i in range(length))
@@ -73,5 +89,6 @@ class FilterModule(object):
             'map_from_pairs': map_from_pairs,
             'es_storage': es_storage,
             'serviceaccount_name': serviceaccount_name,
-            'serviceaccount_namespace': serviceaccount_namespace
+            'serviceaccount_namespace': serviceaccount_namespace,
+            'walk': walk
         }
diff --git a/roles/openshift_logging/filter_plugins/test b/roles/openshift_logging/filter_plugins/test
new file mode 100644
index 000000000..3ad956cca
--- /dev/null
+++ b/roles/openshift_logging/filter_plugins/test
@@ -0,0 +1,34 @@
+import unittest
+from openshift_logging import walk
+
+class TestFilterMethods(unittest.TestCase):
+
+
+    def test_walk_find_key(self):
+        source = {'foo': {'bar.xyz': 'myvalue'}}
+        self.assertEquals(walk(source,'foo#bar.xyz', 123, delimiter='#'), 'myvalue')
+
+
+    def test_walk_return_default(self):
+        source = {'foo': {'bar.xyz': 'myvalue'}}
+        self.assertEquals(walk(source,'foo#bar.abc', 123, delimiter='#'), 123)
+
+
+    def test_walk_limit_max_depth(self):
+        source = {'foo': {'bar.xyz': 'myvalue'}}
+        self.assertEquals(walk(source,'foo#bar.abc#dontfindme', 123, delimiter='#'), 123)
+
+    def test_complex_hash(self):
+        source = {
+            'elasticsearch': {
+                'configmaps': {
+                    'logging-elasticsearch': {
+                        'elasticsearch.yml':  "a string value"
+                    } 
+                }
+            }
+        } 
+        self.assertEquals(walk(source,'elasticsearch#configmaps#logging-elasticsearch#elasticsearch.yml', 123, delimiter='#'), "a string value")
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/roles/openshift_logging/library/openshift_logging_facts.py b/roles/openshift_logging/library/openshift_logging_facts.py
index f10df8da5..98d0d1c4f 100644
--- a/roles/openshift_logging/library/openshift_logging_facts.py
+++ b/roles/openshift_logging/library/openshift_logging_facts.py
@@ -207,7 +207,7 @@ class OpenshiftLoggingFacts(OCBaseCommand):
     def facts_for_configmaps(self, namespace):
         ''' Gathers facts for configmaps in logging namespace '''
         self.default_keys_for("configmaps")
-        a_list = self.oc_command("get", "configmaps", namespace=namespace, add_options=["-l", LOGGING_SELECTOR])
+        a_list = self.oc_command("get", "configmaps", namespace=namespace)
         if len(a_list["items"]) == 0:
             return
         for item in a_list["items"]:
diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml
index 21fd79c28..efc3b97aa 100644
--- a/roles/openshift_logging/tasks/install_logging.yaml
+++ b/roles/openshift_logging/tasks/install_logging.yaml
@@ -78,6 +78,7 @@
     openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_nodeselector if item.0.nodeSelector | default(None) is none else item.0.nodeSelector }}"
     openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_storage_group] if item.0.storageGroups | default([]) | length == 0 else item.0.storageGroups }}"
     _es_containers: "{{item.0.containers}}"
+    _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch#configmaps#logging-elasticsearch#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}"
 
   with_together:
   - "{{ openshift_logging_facts.elasticsearch.deploymentconfigs.values() }}"
@@ -141,7 +142,10 @@
     openshift_logging_es_hostname: "{{ openshift_logging_es_ops_hostname }}"
     openshift_logging_es_edge_term_policy: "{{ openshift_logging_es_ops_edge_term_policy | default('') }}"
     openshift_logging_es_allow_external: "{{ openshift_logging_es_ops_allow_external }}"
+    openshift_logging_es_number_of_shards: "{{ openshift_logging_es_ops_number_of_shards | default(None) }}"
+    openshift_logging_es_number_of_replicas: "{{ openshift_logging_es_ops_number_of_replicas | default(None) }}"
     _es_containers: "{{item.0.containers}}"
+    _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch_ops#configmaps#logging-elasticsearch-ops#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}"
 
   with_together:
   - "{{ openshift_logging_facts.elasticsearch_ops.deploymentconfigs.values() }}"
diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml
index aeff2d198..44f6b00f3 100644
--- a/roles/openshift_logging_elasticsearch/tasks/main.yaml
+++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml
@@ -164,13 +164,17 @@
   when: es_logging_contents is undefined
   changed_when: no
 
+- set_fact:
+    __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}"
+    __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}"
+
 - template:
     src: elasticsearch.yml.j2
     dest: "{{ tempdir }}/elasticsearch.yml"
   vars:
     allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}"
-    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}"
-    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(0) }}"
+    es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}"
+    es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}"
     es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}"
 
   when: es_config_contents is undefined