Merge pull request #4769 from jarrpa/glusterfs-fixes-five

GlusterFS fixes 5

9 files changed, 48 insertions, 18 deletions

diff --git a/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml b/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
index c504bfb80..c2954fde1 100644
--- a/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
+++ b/roles/openshift_hosted/tasks/registry/storage/glusterfs.yml
@@ -35,7 +35,7 @@
   mount:
     state: mounted
     fstype: glusterfs
-    src: "{% if 'glusterfs_registry' in groups %}{{ groups.glusterfs_registry[0] }}{% else %}{{ groups.glusterfs[0] }}{% endif %}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
+    src: "{% if 'glusterfs_registry' in groups %}{% set node = groups.glusterfs_registry[0] %}{% else %}{% set node = groups.glusterfs[0] %}{% endif %}{% if 'glusterfs_hostname' in hostvars[node] %}{{ hostvars[node].glusterfs_hostname }}{% elif 'openshift' in hostvars[node] %}{{ hostvars[node].openshift.node.nodename }}{% else %}{{ node }}{% endif %}:/{{ openshift.hosted.registry.storage.glusterfs.path }}"
     name: "{{ mktemp.stdout }}"
 
 - name: Set registry volume permissions
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md
index 7f4738f1c..b367e7daf 100644
--- a/roles/openshift_storage_glusterfs/README.md
+++ b/roles/openshift_storage_glusterfs/README.md
@@ -64,7 +64,7 @@ their configuration as GlusterFS nodes:
 |--------------------|---------------------------|-----------------------------------------|
 | glusterfs_cluster  | 1                         | The ID of the cluster this node should belong to. This is useful when a single heketi service is expected to manage multiple distinct clusters. **NOTE:** For natively-hosted clusters, all pods will be in the same OpenShift namespace
 | glusterfs_hostname | openshift.node.nodename   | A hostname (or IP address) that will be used for internal GlusterFS communication
-| glusterfs_ip       | openshift.common.ip       | An IP address that will be used by pods to communicate with the GlusterFS node
+| glusterfs_ip       | openshift.common.ip       | An IP address that will be used by pods to communicate with the GlusterFS node. **NOTE:** Required for external GlusterFS nodes
 | glusterfs_zone     | 1                         | A zone number for the node. Zones are used within the cluster for determining how to distribute the bricks of GlusterFS volumes. heketi will try to spread each volumes' bricks as evenly as possible across all zones
 
 Role Variables
@@ -76,7 +76,7 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | Name                                             | Default value           | Description                             |
 |--------------------------------------------------|-------------------------|-----------------------------------------|
 | openshift_storage_glusterfs_timeout              | 300                     | Seconds to wait for pods to become ready
-| openshift_storage_glusterfs_namespace            | 'default'               | Namespace in which to create GlusterFS resources
+| openshift_storage_glusterfs_namespace            | 'glusterfs'             | Namespace in which to create GlusterFS resources
 | openshift_storage_glusterfs_is_native            | True                    | GlusterFS should be containerized
 | openshift_storage_glusterfs_name                 | 'storage'               | A name to identify the GlusterFS cluster, which will be used in resource names
 | openshift_storage_glusterfs_nodeselector         | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name
@@ -85,6 +85,7 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | openshift_storage_glusterfs_version              | 'latest'                | Container image version to use for GlusterFS pods
 | openshift_storage_glusterfs_wipe                 | False                   | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.**
 | openshift_storage_glusterfs_heketi_is_native     | True                    | heketi should be containerized
+| openshift_storage_glusterfs_heketi_cli           | 'heketi-cli'            | Command/Path to invoke the heketi-cli tool **NOTE:** Change this only for **non-native heketi** if heketi-cli is not in the global `$PATH` of the machine running openshift-ansible
 | openshift_storage_glusterfs_heketi_image         | 'heketi/heketi'         | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7'
 | openshift_storage_glusterfs_heketi_version       | 'latest'                | Container image version to use for heketi pods
 | openshift_storage_glusterfs_heketi_admin_key     | auto-generated          | String to use as secret key for performing heketi commands as admin
@@ -108,7 +109,7 @@ are an exception:
 
 | Name                                                  | Default value         | Description                             |
 |-------------------------------------------------------|-----------------------|-----------------------------------------|
-| openshift_storage_glusterfs_registry_namespace        | registry namespace    | Default is to use the hosted registry's namespace, otherwise 'default'
+| openshift_storage_glusterfs_registry_namespace        | registry namespace    | Default is to use the hosted registry's namespace, otherwise 'glusterfs'
 | openshift_storage_glusterfs_registry_name             | 'registry'            | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters
 | openshift_storage_glusterfs_registry_storageclass     | False                 | It is recommended to not create a StorageClass for GlusterFS clusters serving registry storage, so as to avoid performance penalties
 | openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated        | Separate from the above
diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml
index 88e122f55..a846889ca 100644
--- a/roles/openshift_storage_glusterfs/defaults/main.yml
+++ b/roles/openshift_storage_glusterfs/defaults/main.yml
@@ -1,6 +1,6 @@
 ---
 openshift_storage_glusterfs_timeout: 300
-openshift_storage_glusterfs_namespace: 'default'
+openshift_storage_glusterfs_namespace: 'glusterfs'
 openshift_storage_glusterfs_is_native: True
 openshift_storage_glusterfs_name: 'storage'
 openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host"
@@ -8,9 +8,10 @@ openshift_storage_glusterfs_storageclass: True
 openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}"
 openshift_storage_glusterfs_version: 'latest'
 openshift_storage_glusterfs_wipe: False
-openshift_storage_glusterfs_heketi_is_native: True
+openshift_storage_glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_is_native }}"
 openshift_storage_glusterfs_heketi_is_missing: True
 openshift_storage_glusterfs_heketi_deploy_is_missing: True
+openshift_storage_glusterfs_heketi_cli: 'heketi-cli'
 openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}"
 openshift_storage_glusterfs_heketi_version: 'latest'
 openshift_storage_glusterfs_heketi_admin_key: "{{ omit }}"
@@ -26,7 +27,7 @@ openshift_storage_glusterfs_heketi_ssh_sudo: False
 openshift_storage_glusterfs_heketi_ssh_keyfile: '/dev/null'
 
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
-openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
+openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default(openshift_storage_glusterfs_namespace) }}"
 openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}"
 openshift_storage_glusterfs_registry_name: 'registry'
 openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host"
@@ -34,9 +35,10 @@ openshift_storage_glusterfs_registry_storageclass: False
 openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}"
 openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}"
 openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}"
-openshift_storage_glusterfs_registry_heketi_is_native: "{{ openshift_storage_glusterfs_heketi_is_native }}"
+openshift_storage_glusterfs_registry_heketi_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}"
 openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing }}"
 openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}"
+openshift_storage_glusterfs_registry_heketi_cli: "{{ openshift_storage_glusterfs_heketi_cli }}"
 openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}"
 openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}"
 openshift_storage_glusterfs_registry_heketi_admin_key: "{{ omit }}"
@@ -44,9 +46,9 @@ openshift_storage_glusterfs_registry_heketi_user_key: "{{ omit }}"
 openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
 openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
 openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"
-openshift_storage_glusterfs_registry_heketi_port: 8080
-openshift_storage_glusterfs_registry_heketi_executor: 'kubernetes'
-openshift_storage_glusterfs_registry_heketi_ssh_port: 22
-openshift_storage_glusterfs_registry_heketi_ssh_user: 'root'
-openshift_storage_glusterfs_registry_heketi_ssh_sudo: False
-openshift_storage_glusterfs_registry_heketi_ssh_keyfile: '/dev/null'
+openshift_storage_glusterfs_registry_heketi_port: "{{ openshift_storage_glusterfs_heketi_port }}"
+openshift_storage_glusterfs_registry_heketi_executor: "{{ openshift_storage_glusterfs_heketi_executor }}"
+openshift_storage_glusterfs_registry_heketi_ssh_port: "{{ openshift_storage_glusterfs_heketi_ssh_port }}"
+openshift_storage_glusterfs_registry_heketi_ssh_user: "{{ openshift_storage_glusterfs_heketi_ssh_user }}"
+openshift_storage_glusterfs_registry_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo }}"
+openshift_storage_glusterfs_registry_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index 99ad029da..600d8f676 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -1,4 +1,16 @@
 ---
+- name: Make sure heketi-client is installed
+  package: name=heketi-client state=present
+  when:
+  - not openshift.common.is_atomic | bool
+  - not glusterfs_heketi_is_native | bool
+
+- name: Verify heketi-cli is installed
+  shell: "command -v {{ glusterfs_heketi_cli }} >/dev/null 2>&1 || { echo >&2 'ERROR: Make sure heketi-cli is available, then re-run the installer'; exit 1; }"
+  changed_when: False
+  when:
+  - not glusterfs_heketi_is_native | bool
+
 - name: Verify target namespace exists
   oc_project:
     state: present
@@ -173,7 +185,7 @@
 
 - name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
+    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}{{ glusterfs_heketi_cli }} -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin {% if glusterfs_heketi_admin_key is defined %}--secret '{{ glusterfs_heketi_admin_key }}'{% endif %}"
 
 - name: Verify heketi service
   command: "{{ glusterfs_heketi_client }} cluster list"
@@ -203,6 +215,7 @@
       data: "{{ glusterfs_heketi_admin_key }}"
   when:
   - glusterfs_storageclass
+  - glusterfs_heketi_admin_key is defined
 
 - name: Get heketi route
   oc_obj:
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
index 76611d936..b54a8e36c 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
@@ -12,6 +12,7 @@
     glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_heketi_is_native }}"
     glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing }}"
     glusterfs_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}"
+    glusterfs_heketi_cli: "{{ openshift_storage_glusterfs_heketi_cli }}"
     glusterfs_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}"
     glusterfs_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}"
     glusterfs_heketi_admin_key: "{{ openshift_storage_glusterfs_heketi_admin_key }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
index 280088fe4..0b4d1c82b 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -12,6 +12,7 @@
     glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_registry_heketi_is_native }}"
     glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_registry_heketi_is_missing }}"
     glusterfs_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_registry_heketi_deploy_is_missing }}"
+    glusterfs_heketi_cli: "{{ openshift_storage_glusterfs_registry_heketi_cli }}"
     glusterfs_heketi_image: "{{ openshift_storage_glusterfs_registry_heketi_image }}"
     glusterfs_heketi_version: "{{ openshift_storage_glusterfs_registry_heketi_version }}"
     glusterfs_heketi_admin_key: "{{ openshift_storage_glusterfs_registry_heketi_admin_key }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
index 5ef4b5c83..37d3e6ba2 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -126,7 +126,7 @@
 
 - name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_client: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
+    glusterfs_heketi_client: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {{ glusterfs_heketi_cli }} -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
 
 - name: Verify heketi service
   command: "{{ glusterfs_heketi_client }} cluster list"
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
index 2ec9a9e9a..095fb780f 100644
--- a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
@@ -7,5 +7,7 @@ provisioner: kubernetes.io/glusterfs
 parameters:
   resturl: "http://{% if glusterfs_heketi_is_native %}{{ glusterfs_heketi_route }}{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %}"
   restuser: "admin"
+{% if glusterfs_heketi_admin_key is defined %}
   secretNamespace: "{{ glusterfs_namespace }}"
   secretName: "heketi-{{ glusterfs_name }}-admin-secret"
+{%- endif -%}
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2
index 3aac68e2f..d6c28f6dd 100644
--- a/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2
@@ -17,10 +17,20 @@
           "node": {
             "hostnames": {
               "manage": [
-                "{{ hostvars[node].glusterfs_hostname | default(hostvars[node].openshift.node.nodename) }}"
+{%- if 'glusterfs_hostname' in hostvars[node] -%}
+                "{{ hostvars[node].glusterfs_hostname }}"
+{%- elif 'openshift' in hostvars[node] -%}
+                "{{ hostvars[node].openshift.node.nodename }}"
+{%- else -%}
+                "{{ node }}"
+{%- endif -%}
               ],
               "storage": [
-                "{{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}"
+{%- if 'glusterfs_ip' in hostvars[node] -%}
+                "{{ hostvars[node].glusterfs_ip }}"
+{%- else -%}
+                "{{ hostvars[node].openshift.common.ip }}"
+{%- endif -%}
               ]
             },
             "zone": {{ hostvars[node].glusterfs_zone | default(1) }}