GlusterFS can now be run more than once. Ability to add devices to nodes

4 files changed, 48 insertions, 39 deletions

diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
index 9ebb0d5ec..7b705c2d4 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
@@ -85,8 +85,6 @@ objects:
           volumeMounts:
           - name: db
             mountPath: /var/lib/heketi
-          - name: topology
-            mountPath: ${TOPOLOGY_PATH}
           - name: config
             mountPath: /etc/heketi
           readinessProbe:
@@ -103,9 +101,6 @@ objects:
               port: 8080
         volumes:
         - name: db
-        - name: topology
-          secret:
-            secretName: heketi-${CLUSTER_NAME}-topology-secret
         - name: config
           secret:
             secretName: heketi-${CLUSTER_NAME}-config-secret
@@ -138,6 +133,3 @@ parameters:
   displayName: GlusterFS cluster name
   description: A unique name to identify this heketi service, useful for running multiple heketi instances
   value: glusterfs
-- name: TOPOLOGY_PATH
-  displayName: heketi topology file location
-  required: True
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index a31c5bd5e..a43708d71 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -50,8 +50,8 @@
     kind: pod
     state: list
     selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
-  register: heketi_pod
-  until: "heketi_pod.results.results[0]['items'] | count == 0"
+  register: deploy_heketi_pod
+  until: "deploy_heketi_pod.results.results[0]['items'] | count == 0"
   delay: 10
   retries: "{{ (glusterfs_timeout | int / 10) | int }}"
   when: glusterfs_heketi_wipe
@@ -102,7 +102,7 @@
     state: list
     kind: pod
     selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
-  register: heketi_pod
+  register: deploy_heketi_pod
   when: glusterfs_heketi_is_native
 
 - name: Check if need to deploy deploy-heketi
@@ -110,9 +110,9 @@
     glusterfs_heketi_deploy_is_missing: False
   when:
   - "glusterfs_heketi_is_native"
-  - "heketi_pod.results.results[0]['items'] | count > 0"
+  - "deploy_heketi_pod.results.results[0]['items'] | count > 0"
   # deploy-heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
-  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+  - "deploy_heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
 
 - name: Check for existing heketi pod
   oc_obj:
@@ -146,6 +146,21 @@
   when:
   - glusterfs_heketi_is_native
 
+- name: Get heketi admin secret
+  oc_secret:
+    state: list
+    namespace: "{{ glusterfs_namespace }}"
+    name: "heketi-{{ glusterfs_name }}-admin-secret"
+    decode: True
+  register: glusterfs_heketi_admin_secret
+
+- name: Set heketi admin key
+  set_fact:
+    glusterfs_heketi_admin_key: "{{ glusterfs_heketi_admin_secret.results.decoded.key }}"
+  when:
+  - glusterfs_is_native
+  - glusterfs_heketi_admin_secret.results.results[0]
+
 - name: Generate heketi admin key
   set_fact:
     glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
@@ -189,14 +204,37 @@
   - glusterfs_heketi_deploy_is_missing
   - glusterfs_heketi_is_missing
 
+- name: Wait for deploy-heketi pod
+  oc_obj:
+    namespace: "{{ glusterfs_namespace }}"
+    kind: pod
+    state: list
+    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
+  register: deploy_heketi_pod
+  until:
+  - "deploy_heketi_pod.results.results[0]['items'] | count > 0"
+  # Pod's 'Ready' status must be True
+  - "deploy_heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
+  delay: 10
+  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
+  when:
+  - glusterfs_heketi_is_native
+  - not glusterfs_heketi_deploy_is_missing
+  - glusterfs_heketi_is_missing
+
 - name: Set heketi-cli command
   set_fact:
-    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}{{ glusterfs_heketi_cli }} -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin {% if glusterfs_heketi_admin_key is defined %}--secret '{{ glusterfs_heketi_admin_key }}'{% endif %}"
+    glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {%if not glusterfs_heketi_is_missing %}{{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }}{% else %}{{ deploy_heketi_pod.results.results[0]['items'][0]['metadata']['name'] }}{% endif %} {% endif %}{{ glusterfs_heketi_cli }} -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin {% if glusterfs_heketi_admin_key is defined %}--secret '{{ glusterfs_heketi_admin_key }}'{% endif %}"
 
 - name: Verify heketi service
   command: "{{ glusterfs_heketi_client }} cluster list"
   changed_when: False
 
+- name: Place heketi topology on heketi Pod
+  shell: "{{ openshift.common.client_binary }} exec --namespace={{ glusterfs_namespace }} -i {%if not glusterfs_heketi_is_missing %}{{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }}{% else %}{{ deploy_heketi_pod.results.results[0]['items'][0]['metadata']['name'] }}{% endif %} -- bash -c 'mkdir -p {{ mktemp.stdout }} && cat > {{ mktemp.stdout }}/topology.json' < {{ mktemp.stdout }}/topology.json"
+  when:
+  - glusterfs_heketi_is_native
+
 - name: Load heketi topology
   command: "{{ glusterfs_heketi_client }} topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
   register: topology_load
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
index 3ba1eb2d2..73396c9af 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
@@ -6,16 +6,6 @@
   with_items:
   - "deploy-heketi-template.yml"
 
-- name: Create heketi topology secret
-  oc_secret:
-    namespace: "{{ glusterfs_namespace }}"
-    state: present
-    name: "heketi-{{ glusterfs_name }}-topology-secret"
-    force: True
-    files:
-    - name: topology.json
-      path: "{{ mktemp.stdout }}/topology.json"
-
 - name: Create deploy-heketi template
   oc_obj:
     namespace: "{{ glusterfs_namespace }}"
@@ -39,18 +29,7 @@
       HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
       CLUSTER_NAME: "{{ glusterfs_name }}"
-      TOPOLOGY_PATH: "{{ mktemp.stdout }}"
 
-- name: Wait for deploy-heketi pod
-  oc_obj:
-    namespace: "{{ glusterfs_namespace }}"
-    kind: pod
-    state: list
-    selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
-  register: heketi_pod
-  until:
-  - "heketi_pod.results.results[0]['items'] | count > 0"
-  # Pod's 'Ready' status must be True
-  - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
-  delay: 10
-  retries: "{{ (glusterfs_timeout | int / 10) | int }}"
+- name: Set heketi Deployed fact
+  set_fact:
+    glusterfs_heketi_deploy_is_missing: False
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
index afc04a537..54a6dd7c3 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -4,7 +4,7 @@
   register: setup_storage
 
 - name: Copy heketi-storage list
-  shell: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
+  shell: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ deploy_heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
 
 # This is used in the subsequent task
 - name: Copy the admin client config