Merge branch 'master' into mbruzek-openshift-openstack

11 files changed, 159 insertions, 30 deletions

diff --git a/roles/openshift_logging/tasks/annotate_ops_projects.yaml b/roles/openshift_logging/tasks/annotate_ops_projects.yaml
index 59d6098d4..4a2ee64f0 100644
--- a/roles/openshift_logging/tasks/annotate_ops_projects.yaml
+++ b/roles/openshift_logging/tasks/annotate_ops_projects.yaml
@@ -1,6 +1,6 @@
 ---
 - command: >
-    {{ openshift.common.client_binary }}
+    {{ openshift_client_binary }}
     --config={{ openshift.common.config_base }}/master/admin.kubeconfig
     get namespaces -o jsonpath={.items[*].metadata.name} {{ __default_logging_ops_projects | join(' ') }}
   register: __logging_ops_projects
diff --git a/roles/openshift_logging/tasks/delete_logging.yaml b/roles/openshift_logging/tasks/delete_logging.yaml
index af36d67c6..fbc3e3fd1 100644
--- a/roles/openshift_logging/tasks/delete_logging.yaml
+++ b/roles/openshift_logging/tasks/delete_logging.yaml
@@ -109,14 +109,14 @@
 
 # remove annotations added by logging
 - command: >
-    {{ openshift.common.client_binary }}
+    {{ openshift_client_binary }}
     --config={{ openshift.common.config_base }}/master/admin.kubeconfig
     get namespaces -o name {{ __default_logging_ops_projects | join(' ') }}
   register: __logging_ops_projects
 
 - name: Remove Annotation of Operations Projects
   command: >
-    {{ openshift.common.client_binary }}
+    {{ openshift_client_binary }}
     --config={{ openshift.common.config_base }}/master/admin.kubeconfig
     annotate {{ project }} openshift.io/logging.ui.hostname-
   with_items: "{{ __logging_ops_projects.stdout_lines }}"
@@ -126,7 +126,18 @@
     - __logging_ops_projects.stderr | length == 0
 
 ## EventRouter
-- include_role:
+- import_role:
     name: openshift_logging_eventrouter
   when:
     not openshift_logging_install_eventrouter | default(false) | bool
+
+# Update asset config in openshift-web-console namespace
+- name: Remove Kibana route information from web console asset config
+  include_role:
+    name: openshift_web_console
+    tasks_from: update_asset_config.yml
+  vars:
+    asset_config_edits:
+      - key: loggingPublicURL
+        value: ""
+  when: openshift_web_console_install | default(true) | bool
diff --git a/roles/openshift_logging/tasks/generate_certs.yaml b/roles/openshift_logging/tasks/generate_certs.yaml
index 082c0128f..0d7f8c056 100644
--- a/roles/openshift_logging/tasks/generate_certs.yaml
+++ b/roles/openshift_logging/tasks/generate_certs.yaml
@@ -17,7 +17,7 @@
 
 - name: Generate certificates
   command: >
-    {{ openshift.common.client_binary }} adm --config={{ mktemp.stdout }}/admin.kubeconfig ca create-signer-cert
+    {{ openshift_client_binary }} adm --config={{ mktemp.stdout }}/admin.kubeconfig ca create-signer-cert
     --key={{generated_certs_dir}}/ca.key --cert={{generated_certs_dir}}/ca.crt
     --serial={{generated_certs_dir}}/ca.serial.txt --name=logging-signer-test
   check_mode: no
@@ -139,10 +139,10 @@
 
 # TODO: make idempotent
 - name: Generate proxy session
-  set_fact: session_secret={{ 200 | oo_random_word}}
+  set_fact: session_secret={{ 200 | lib_utils_oo_random_word}}
   check_mode: no
 
 # TODO: make idempotent
 - name: Generate oauth client secret
-  set_fact: oauth_secret={{ 64 | oo_random_word}}
+  set_fact: oauth_secret={{ 64 | lib_utils_oo_random_word}}
   check_mode: no
diff --git a/roles/openshift_logging/tasks/generate_jks.yaml b/roles/openshift_logging/tasks/generate_jks.yaml
index d6ac88dcc..6e3204589 100644
--- a/roles/openshift_logging/tasks/generate_jks.yaml
+++ b/roles/openshift_logging/tasks/generate_jks.yaml
@@ -24,25 +24,21 @@
   local_action: file path="{{local_tmp.stdout}}/elasticsearch.jks" state=touch mode="u=rw,g=r,o=r"
   when: elasticsearch_jks.stat.exists
   changed_when: False
-  become: no
 
 - name: Create placeholder for previously created JKS certs to prevent recreating...
   local_action: file path="{{local_tmp.stdout}}/logging-es.jks" state=touch mode="u=rw,g=r,o=r"
   when: logging_es_jks.stat.exists
   changed_when: False
-  become: no
 
 - name: Create placeholder for previously created JKS certs to prevent recreating...
   local_action: file path="{{local_tmp.stdout}}/system.admin.jks" state=touch mode="u=rw,g=r,o=r"
   when: system_admin_jks.stat.exists
   changed_when: False
-  become: no
 
 - name: Create placeholder for previously created JKS certs to prevent recreating...
   local_action: file path="{{local_tmp.stdout}}/truststore.jks" state=touch mode="u=rw,g=r,o=r"
   when: truststore_jks.stat.exists
   changed_when: False
-  become: no
 
 - name: pulling down signing items from host
   fetch:
@@ -61,12 +57,10 @@
   vars:
     - top_dir: "{{local_tmp.stdout}}"
   when: not elasticsearch_jks.stat.exists or not logging_es_jks.stat.exists or not system_admin_jks.stat.exists or not truststore_jks.stat.exists
-  become: no
 
 - name: Run JKS generation script
   local_action: script generate-jks.sh {{local_tmp.stdout}} {{openshift_logging_namespace}}
   check_mode: no
-  become: no
   when: not elasticsearch_jks.stat.exists or not logging_es_jks.stat.exists or not system_admin_jks.stat.exists or not truststore_jks.stat.exists
 
 - name: Pushing locally generated JKS certs to remote host...
diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml
index bb8ebec6b..f82e55b98 100644
--- a/roles/openshift_logging/tasks/install_logging.yaml
+++ b/roles/openshift_logging/tasks/install_logging.yaml
@@ -1,9 +1,12 @@
 ---
 - name: Gather OpenShift Logging Facts
   openshift_logging_facts:
-    oc_bin: "{{openshift.common.client_binary}}"
+    oc_bin: "{{openshift_client_binary}}"
     openshift_logging_namespace: "{{openshift_logging_namespace}}"
 
+## This is include vs import because we need access to group/inventory variables
+- include_tasks: set_defaults_from_current.yml
+
 - name: Set logging project
   oc_project:
     state: present
@@ -84,14 +87,14 @@
 
     openshift_logging_elasticsearch_storage_type: "{{ elasticsearch_storage_type }}"
     openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_pv_selector }}"
-    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_pvc_storage_class_name }}"
+    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_pvc_storage_class_name | default() }}"
     openshift_logging_elasticsearch_nodeselector: "{{ openshift_logging_es_nodeselector if outer_item.0.nodeSelector | default(None) is none else outer_item.0.nodeSelector }}"
     openshift_logging_elasticsearch_storage_group: "{{ [openshift_logging_es_storage_group] if outer_item.0.storageGroups | default([]) | length == 0 else outer_item.0.storageGroups }}"
     _es_containers: "{{ outer_item.0.containers}}"
     _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch#configmaps#logging-elasticsearch#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}"
 
   with_together:
-  - "{{ openshift_logging_facts.elasticsearch.deploymentconfigs.values() }}"
+  - "{{ openshift_logging_facts.elasticsearch.deploymentconfigs.values() | list }}"
   - "{{ openshift_logging_facts.elasticsearch.pvcs }}"
   - "{{ es_indices }}"
   loop_control:
@@ -111,7 +114,7 @@
 
     openshift_logging_elasticsearch_storage_type: "{{ elasticsearch_storage_type }}"
     openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_pv_selector }}"
-    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_pvc_storage_class_name }}"
+    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_pvc_storage_class_name | default() }}"
 
   with_sequence: count={{ openshift_logging_es_cluster_size | int - openshift_logging_facts.elasticsearch.deploymentconfigs.keys() | count }}
   loop_control:
@@ -148,7 +151,7 @@
     openshift_logging_elasticsearch_pvc_size: "{{ openshift_logging_es_ops_pvc_size }}"
     openshift_logging_elasticsearch_pvc_dynamic: "{{ openshift_logging_es_ops_pvc_dynamic }}"
     openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_ops_pv_selector }}"
-    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_ops_pvc_storage_class_name }}"
+    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_ops_pvc_storage_class_name | default() }}"
     openshift_logging_elasticsearch_memory_limit: "{{ openshift_logging_es_ops_memory_limit }}"
     openshift_logging_elasticsearch_cpu_limit: "{{ openshift_logging_es_ops_cpu_limit }}"
     openshift_logging_elasticsearch_cpu_request: "{{ openshift_logging_es_ops_cpu_request }}"
@@ -166,7 +169,7 @@
     _es_configmap: "{{ openshift_logging_facts | walk('elasticsearch_ops#configmaps#logging-elasticsearch-ops#elasticsearch.yml', '{}', delimiter='#') | from_yaml }}"
 
   with_together:
-  - "{{ openshift_logging_facts.elasticsearch_ops.deploymentconfigs.values() }}"
+  - "{{ openshift_logging_facts.elasticsearch_ops.deploymentconfigs.values() | list }}"
   - "{{ openshift_logging_facts.elasticsearch_ops.pvcs }}"
   - "{{ es_ops_indices }}"
   loop_control:
@@ -190,7 +193,7 @@
     openshift_logging_elasticsearch_pvc_size: "{{ openshift_logging_es_ops_pvc_size }}"
     openshift_logging_elasticsearch_pvc_dynamic: "{{ openshift_logging_es_ops_pvc_dynamic }}"
     openshift_logging_elasticsearch_pvc_pv_selector: "{{ openshift_logging_es_ops_pv_selector }}"
-    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_ops_pvc_storage_class_name }}"
+    openshift_logging_elasticsearch_pvc_storage_class_name: "{{ openshift_logging_es_ops_pvc_storage_class_name | default() }}"
     openshift_logging_elasticsearch_memory_limit: "{{ openshift_logging_es_ops_memory_limit }}"
     openshift_logging_elasticsearch_cpu_limit: "{{ openshift_logging_es_ops_cpu_limit }}"
     openshift_logging_elasticsearch_cpu_request: "{{ openshift_logging_es_ops_cpu_request }}"
@@ -210,7 +213,7 @@
 
 
 ## Kibana
-- include_role:
+- import_role:
     name: openshift_logging_kibana
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -223,7 +226,7 @@
     openshift_logging_kibana_image_pull_secret: "{{ openshift_logging_image_pull_secret }}"
 
 
-- include_role:
+- import_role:
     name: openshift_logging_kibana
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -253,7 +256,7 @@
 - include_tasks: annotate_ops_projects.yaml
 
 ## Curator
-- include_role:
+- import_role:
     name: openshift_logging_curator
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -263,7 +266,7 @@
     openshift_logging_curator_master_url: "{{ openshift_logging_master_url }}"
     openshift_logging_curator_image_pull_secret: "{{ openshift_logging_image_pull_secret }}"
 
-- include_role:
+- import_role:
     name: openshift_logging_curator
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -281,7 +284,7 @@
   - openshift_logging_use_ops | bool
 
 ## Mux
-- include_role:
+- import_role:
     name: openshift_logging_mux
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -294,7 +297,7 @@
 
 
 ## Fluentd
-- include_role:
+- import_role:
     name: openshift_logging_fluentd
   vars:
     generated_certs_dir: "{{openshift.common.config_base}}/logging"
@@ -305,10 +308,27 @@
 
 
 ## EventRouter
-- include_role:
+- import_role:
     name: openshift_logging_eventrouter
   when:
     openshift_logging_install_eventrouter | default(false) | bool
 
 
+# TODO: Remove when asset config is removed from master-config.yaml
 - include_tasks: update_master_config.yaml
+
+# Update asset config in openshift-web-console namespace
+- name: Add Kibana route information to web console asset config
+  include_role:
+    name: openshift_web_console
+    tasks_from: update_console_config.yml
+  vars:
+    console_config_edits:
+    - key: clusterInfo#loggingPublicURL
+      value: "https://{{ openshift_logging_kibana_hostname }}"
+    # Continue to set the old deprecated property until the
+    # origin-web-console image is updated for the new name.
+    # This will be removed in a future pull.
+    - key: loggingPublicURL
+      value: "https://{{ openshift_logging_kibana_hostname }}"
+  when: openshift_web_console_install | default(true) | bool
diff --git a/roles/openshift_logging/tasks/main.yaml b/roles/openshift_logging/tasks/main.yaml
index 9949bb95d..60cc399fa 100644
--- a/roles/openshift_logging/tasks/main.yaml
+++ b/roles/openshift_logging/tasks/main.yaml
@@ -17,7 +17,11 @@
   register: local_tmp
   changed_when: False
   check_mode: no
-  become: no
+
+- name: Chmod local temp directory for doing work in
+  local_action: command chmod 777 "{{ local_tmp.stdout }}"
+  changed_when: False
+  check_mode: no
 
 - include_tasks: install_logging.yaml
   when:
@@ -31,4 +35,3 @@
   local_action: file path="{{local_tmp.stdout}}" state=absent
   tags: logging_cleanup
   changed_when: False
-  become: no
diff --git a/roles/openshift_logging/tasks/patch_configmap_file.yaml b/roles/openshift_logging/tasks/patch_configmap_file.yaml
new file mode 100644
index 000000000..30087fe6a
--- /dev/null
+++ b/roles/openshift_logging/tasks/patch_configmap_file.yaml
@@ -0,0 +1,35 @@
+---
+## The purpose of this task file is to get a patch that is based on the diff
+##  between configmap_current_file and configmap_new_file. The module
+##  logging_patch takes the paths of two files to compare and also a list of
+##  variables whose line we exclude from the diffs.
+##  We then patch the new configmap file so that we can build a configmap
+##  using that file later. We then use oc apply to idempotenly modify any
+##  existing configmap.
+
+## The following variables are expected to be provided when including this task:
+# __configmap_output         -- This is provided to us from patch_configmap_files.yaml
+#                                it is a dict of the configmap where configmap_current_file exists
+# configmap_current_file     -- The name of the data file in the __configmap_output
+# configmap_new_file         -- The path to the file that we intend to oc apply later
+#                                we apply our generated patch to this file.
+# configmap_protected_lines  -- The list of variables to exclude from the diff
+
+- copy:
+    content: "{{ __configmap_output.results.results[0]['data'][configmap_current_file] }}"
+    dest: "{{ tempdir }}/current.yml"
+
+- logging_patch:
+    original_file: "{{ tempdir }}/current.yml"
+    new_file: "{{ configmap_new_file }}"
+    whitelist: "{{ configmap_protected_lines | default([]) }}"
+  register: patch_output
+
+- copy:
+    content: "{{ patch_output.raw_patch }}\n"
+    dest: "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0
+
+- command: >
+    patch --force --quiet -u "{{ configmap_new_file }}" "{{ tempdir }}/patch.patch"
+  when: patch_output.raw_patch | length > 0
diff --git a/roles/openshift_logging/tasks/patch_configmap_files.yaml b/roles/openshift_logging/tasks/patch_configmap_files.yaml
new file mode 100644
index 000000000..74a9cc287
--- /dev/null
+++ b/roles/openshift_logging/tasks/patch_configmap_files.yaml
@@ -0,0 +1,31 @@
+---
+## The purpose of this task file is to take in a list of configmap files provided
+##  in the variable configmap_file_names, which correspond to the data sections
+##  within a configmap. We iterate over each of these files and create a patch
+##  from the diff between current_file and new_file to try to maintain any custom
+##  changes that a user may have made to a currently deployed configmap while
+##  trying to idempotently update with any role provided files.
+
+## The following variables are expected to be provided when including this task:
+# configmap_name        -- This is the name of the configmap that the files exist in
+# configmap_namespace   -- The namespace that the configmap lives in
+# configmap_file_names  -- This is expected to be passed in as a dict
+#   current_file        -- The name of the data entry within the configmap
+#   new_file            -- The file path to the file we are comparing to current_file
+#   protected_lines     -- List of variables whose line will be excluded when creating a diff
+
+- oc_configmap:
+    name: "{{ configmap_name }}"
+    state: list
+    namespace: "{{ configmap_namespace }}"
+  register: __configmap_output
+
+- when: __configmap_output.results.stderr is undefined
+  include_tasks: patch_configmap_file.yaml
+  vars:
+    configmap_current_file: "{{ configmap_files.current_file }}"
+    configmap_new_file: "{{ configmap_files.new_file }}"
+    configmap_protected_lines: "{{ configmap_files.protected_lines | default([]) }}"
+  with_items: "{{ configmap_file_names }}"
+  loop_control:
+    loop_var: configmap_files
diff --git a/roles/openshift_logging/tasks/procure_server_certs.yaml b/roles/openshift_logging/tasks/procure_server_certs.yaml
index 00de0ca06..bc817075d 100644
--- a/roles/openshift_logging/tasks/procure_server_certs.yaml
+++ b/roles/openshift_logging/tasks/procure_server_certs.yaml
@@ -27,7 +27,7 @@
 
 - name: Creating signed server cert and key for {{ cert_info.procure_component }}
   command: >
-     {{ openshift.common.client_binary }} adm --config={{ mktemp.stdout }}/admin.kubeconfig ca create-server-cert
+     {{ openshift_client_binary }} adm --config={{ mktemp.stdout }}/admin.kubeconfig ca create-server-cert
      --key={{generated_certs_dir}}/{{cert_info.procure_component}}.key --cert={{generated_certs_dir}}/{{cert_info.procure_component}}.crt
      --hostnames={{cert_info.hostnames|quote}} --signer-cert={{generated_certs_dir}}/ca.crt --signer-key={{generated_certs_dir}}/ca.key
      --signer-serial={{generated_certs_dir}}/ca.serial.txt
diff --git a/roles/openshift_logging/tasks/set_defaults_from_current.yml b/roles/openshift_logging/tasks/set_defaults_from_current.yml
new file mode 100644
index 000000000..dde362abe
--- /dev/null
+++ b/roles/openshift_logging/tasks/set_defaults_from_current.yml
@@ -0,0 +1,34 @@
+---
+
+## We are pulling default values from configmaps if they exist already
+## Using conditional_set_fact allows us to set the value of a variable based on
+##  the value of another one, if it is already defined. Else we don't set the
+##  left hand side (it stays undefined as well).
+
+## conditional_set_fact allows us to specify a fact source, so first we try to
+##  set variables in the logging-elasticsearch & logging-elasticsearch-ops configmaps
+##  afterwards we set the value of the variable based on the value in the inventory
+##  but fall back to using the value from a configmap as a default. If neither is set
+##  then the variable remains undefined and the role default will be used.
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_number_of_shards: index.number_of_shards
+      __openshift_logging_es_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch']['configmaps']['logging-elasticsearch'] is defined
+
+- conditional_set_fact:
+    facts: "{{ openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops']['elasticsearch.yml'] | flatten_dict }}"
+    vars:
+      __openshift_logging_es_ops_number_of_shards: index.number_of_shards
+      __openshift_logging_es_ops_number_of_replicas: index.number_of_replicas
+  when: openshift_logging_facts['elasticsearch_ops']['configmaps']['logging-elasticsearch-ops'] is defined
+
+- conditional_set_fact:
+    facts: "{{ hostvars[inventory_hostname] }}"
+    vars:
+      openshift_logging_es_number_of_shards: openshift_logging_es_number_of_shards | __openshift_logging_es_number_of_shards
+      openshift_logging_es_number_of_replicas: openshift_logging_es_number_of_replicas | __openshift_logging_es_number_of_replicas
+      openshift_logging_es_ops_number_of_shards: openshift_logging_es_ops_number_of_shards | __openshift_logging_es_ops_number_of_shards
+      openshift_logging_es_ops_number_of_replicas: openshift_logging_es_ops_number_of_replicas | __openshift_logging_es_ops_number_of_replicas
diff --git a/roles/openshift_logging/tasks/update_master_config.yaml b/roles/openshift_logging/tasks/update_master_config.yaml
index b96b8e29d..c0f42ba97 100644
--- a/roles/openshift_logging/tasks/update_master_config.yaml
+++ b/roles/openshift_logging/tasks/update_master_config.yaml
@@ -1,4 +1,5 @@
 ---
+# TODO: Remove when asset config is removed from master-config.yaml
 - name: Adding Kibana route information to loggingPublicURL
   modify_yaml:
     dest: "{{ openshift.common.config_base }}/master/master-config.yaml"