Merge pull request #4720 from jarrpa/glusterfs-fixes-four

GlusterFS Fixes 4

13 files changed, 221 insertions, 2 deletions

diff --git a/inventory/byo/hosts.byo.glusterfs.external.example b/inventory/byo/hosts.byo.glusterfs.external.example
new file mode 100644
index 000000000..140033a39
--- /dev/null
+++ b/inventory/byo/hosts.byo.glusterfs.external.example
@@ -0,0 +1,56 @@
+# This is an example of a bring your own (byo) host inventory for a cluster
+# with natively hosted, containerized GlusterFS storage.
+#
+# This inventory may be used with the byo/config.yml playbook to deploy a new
+# cluster with GlusterFS storage, which will use that storage to create a
+# volume that will provide backend storage for a hosted Docker registry.
+#
+# This inventory may also be used with byo/openshift-glusterfs/config.yml to
+# deploy GlusterFS storage on an existing cluster. With this playbook, the
+# registry backend volume will be created but the administrator must then
+# either deploy a hosted registry or change an existing hosted registry to use
+# that volume.
+#
+# There are additional configuration parameters that can be specified to
+# control the deployment and state of a GlusterFS cluster. Please see the
+# documentation in playbooks/byo/openshift-glusterfs/README.md and
+# roles/openshift_storage_glusterfs/README.md for additional details.
+
+[OSEv3:children]
+masters
+nodes
+# Specify there will be GlusterFS nodes
+glusterfs
+
+[OSEv3:vars]
+ansible_ssh_user=root
+openshift_deployment_type=origin
+# Specify that we want to use an external GlusterFS cluster
+openshift_storage_glusterfs_is_native=False
+# Specify the IP address or hostname of the external heketi service
+openshift_storage_glusterfs_heketi_url=172.0.0.1
+
+[masters]
+master  node=True storage=True master=True
+
+[nodes]
+master  node=True storage=True master=True openshift_schedulable=False
+node0   node=True openshift_schedulable=True
+node1   node=True openshift_schedulable=True
+node2   node=True openshift_schedulable=True
+
+# Specify the glusterfs group, which contains the nodes of the external
+# GlusterFS cluster. At a minimum, each node must have "glusterfs_hostname" 
+# and "glusterfs_devices" variables defined.
+#
+# The first variable indicates the hostname of the external GLusterFS node,
+# and must be reachable by the external heketi service.
+#
+# The second variable is a list of block devices the node will have access to
+# that are intended solely for use as GlusterFS storage. These block devices
+# must be bare (e.g. have no data, not be marked as LVM PVs), and will be
+# formatted.
+[glusterfs]
+node0  glusterfs_devices='[ "/dev/vdb" ]'
+node1  glusterfs_devices='[ "/dev/vdb", "/dev/vdc" ]'
+node2  glusterfs_devices='[ "/dev/vdd" ]'
diff --git a/inventory/byo/hosts.byo.glusterfs.mixed.example b/inventory/byo/hosts.byo.glusterfs.mixed.example
new file mode 100644
index 000000000..96e997dc7
--- /dev/null
+++ b/inventory/byo/hosts.byo.glusterfs.mixed.example
@@ -0,0 +1,59 @@
+# This is an example of a bring your own (byo) host inventory for a cluster
+# with natively hosted, containerized GlusterFS storage.
+#
+# This inventory may be used with the byo/config.yml playbook to deploy a new
+# cluster with GlusterFS storage, which will use that storage to create a
+# volume that will provide backend storage for a hosted Docker registry.
+#
+# This inventory may also be used with byo/openshift-glusterfs/config.yml to
+# deploy GlusterFS storage on an existing cluster. With this playbook, the
+# registry backend volume will be created but the administrator must then
+# either deploy a hosted registry or change an existing hosted registry to use
+# that volume.
+#
+# There are additional configuration parameters that can be specified to
+# control the deployment and state of a GlusterFS cluster. Please see the
+# documentation in playbooks/byo/openshift-glusterfs/README.md and
+# roles/openshift_storage_glusterfs/README.md for additional details.
+
+[OSEv3:children]
+masters
+nodes
+# Specify there will be GlusterFS nodes
+glusterfs
+
+[OSEv3:vars]
+ansible_ssh_user=root
+openshift_deployment_type=origin
+# Specify that we want to use an external GlusterFS cluster and a native
+# heketi service
+openshift_storage_glusterfs_is_native=False
+openshift_storage_glusterfs_heketi_is_native=False
+# Specify that heketi will use SSH to communicate to the GlusterFS nodes and
+# the private key file it will use for authentication
+openshift_storage_glusterfs_heketi_executor=ssh
+openshift_storage_glusterfs_heketi_ssh_keyfile=/root/id_rsa
+[masters]
+master  node=True storage=True master=True
+
+[nodes]
+master  node=True storage=True master=True openshift_schedulable=False
+node0   node=True openshift_schedulable=True
+node1   node=True openshift_schedulable=True
+node2   node=True openshift_schedulable=True
+
+# Specify the glusterfs group, which contains the nodes of the external
+# GlusterFS cluster. At a minimum, each node must have "glusterfs_hostname" 
+# and "glusterfs_devices" variables defined.
+#
+# The first variable indicates the hostname of the external GLusterFS node,
+# and must be reachable by the external heketi service.
+#
+# The second variable is a list of block devices the node will have access to
+# that are intended solely for use as GlusterFS storage. These block devices
+# must be bare (e.g. have no data, not be marked as LVM PVs), and will be
+# formatted.
+[glusterfs]
+node0  glusterfs_devices='[ "/dev/vdb" ]'
+node1  glusterfs_devices='[ "/dev/vdb", "/dev/vdc" ]'
+node2  glusterfs_devices='[ "/dev/vdd" ]'
diff --git a/inventory/byo/hosts.byo.native-glusterfs.example b/inventory/byo/hosts.byo.glusterfs.native.example
index dc847a5b2..dc847a5b2 100644
--- a/inventory/byo/hosts.byo.native-glusterfs.example
+++ b/inventory/byo/hosts.byo.glusterfs.native.example
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md
index 4b9a5f42c..7f4738f1c 100644
--- a/roles/openshift_storage_glusterfs/README.md
+++ b/roles/openshift_storage_glusterfs/README.md
@@ -92,6 +92,11 @@ GlusterFS cluster into a new or existing OpenShift cluster:
 | openshift_storage_glusterfs_heketi_topology_load | True                    | Load the GlusterFS topology information into heketi
 | openshift_storage_glusterfs_heketi_url           | Undefined               | When heketi is native, this sets the hostname portion of the final heketi route URL. When heketi is external, this is the full URL to the heketi service.
 | openshift_storage_glusterfs_heketi_port          | 8080                    | TCP port for external heketi service **NOTE:** This has no effect in native mode
+| openshift_storage_glusterfs_heketi_executor      | 'kubernetes'            | Selects how a native heketi service will manage GlusterFS nodes: 'kubernetes' for native nodes, 'ssh' for external nodes
+| openshift_storage_glusterfs_heketi_ssh_port      | 22                      | SSH port for external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_user      | 'root'                  | SSH user for external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_sudo      | False                   | Whether to sudo (if non-root user) for SSH to external GlusterFS nodes via native heketi
+| openshift_storage_glusterfs_heketi_ssh_keyfile   | '/dev/null'             | Path to a private key file for use with SSH connections to external GlusterFS nodes via native heketi **NOTE:** This must be an absolute path
 | openshift_storage_glusterfs_heketi_wipe          | False                   | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
 
 Each role variable also has a corresponding variable to optionally configure a
diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml
index 4ff56af9e..88e122f55 100644
--- a/roles/openshift_storage_glusterfs/defaults/main.yml
+++ b/roles/openshift_storage_glusterfs/defaults/main.yml
@@ -19,6 +19,11 @@ openshift_storage_glusterfs_heketi_topology_load: True
 openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"
 openshift_storage_glusterfs_heketi_url: "{{ omit }}"
 openshift_storage_glusterfs_heketi_port: 8080
+openshift_storage_glusterfs_heketi_executor: 'kubernetes'
+openshift_storage_glusterfs_heketi_ssh_port: 22
+openshift_storage_glusterfs_heketi_ssh_user: 'root'
+openshift_storage_glusterfs_heketi_ssh_sudo: False
+openshift_storage_glusterfs_heketi_ssh_keyfile: '/dev/null'
 
 openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
 openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
@@ -40,3 +45,8 @@ openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage
 openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
 openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"
 openshift_storage_glusterfs_registry_heketi_port: 8080
+openshift_storage_glusterfs_registry_heketi_executor: 'kubernetes'
+openshift_storage_glusterfs_registry_heketi_ssh_port: 22
+openshift_storage_glusterfs_registry_heketi_ssh_user: 'root'
+openshift_storage_glusterfs_registry_heketi_ssh_sudo: False
+openshift_storage_glusterfs_registry_heketi_ssh_keyfile: '/dev/null'
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
index 4434f750c..9ebb0d5ec 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
@@ -71,7 +71,7 @@ objects:
           - name: HEKETI_ADMIN_KEY
             value: ${HEKETI_ADMIN_KEY}
           - name: HEKETI_EXECUTOR
-            value: kubernetes
+            value: ${HEKETI_EXECUTOR}
           - name: HEKETI_FSTAB
             value: /var/lib/heketi/fstab
           - name: HEKETI_SNAPSHOT_LIMIT
@@ -87,6 +87,8 @@ objects:
             mountPath: /var/lib/heketi
           - name: topology
             mountPath: ${TOPOLOGY_PATH}
+          - name: config
+            mountPath: /etc/heketi
           readinessProbe:
             timeoutSeconds: 3
             initialDelaySeconds: 3
@@ -104,6 +106,9 @@ objects:
         - name: topology
           secret:
             secretName: heketi-${CLUSTER_NAME}-topology-secret
+        - name: config
+          secret:
+            secretName: heketi-${CLUSTER_NAME}-config-secret
 parameters:
 - name: HEKETI_USER_KEY
   displayName: Heketi User Secret
@@ -111,6 +116,10 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_EXECUTOR
+  displayName: heketi executor type
+  description: Set the executor type, kubernetes or ssh
+  value: kubernetes
 - name: HEKETI_KUBE_NAMESPACE
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
index e3fa0a9fb..61b6a8c13 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
@@ -67,7 +67,7 @@ objects:
           - name: HEKETI_ADMIN_KEY
             value: ${HEKETI_ADMIN_KEY}
           - name: HEKETI_EXECUTOR
-            value: kubernetes
+            value: ${HEKETI_EXECUTOR}
           - name: HEKETI_FSTAB
             value: /var/lib/heketi/fstab
           - name: HEKETI_SNAPSHOT_LIMIT
@@ -81,6 +81,8 @@ objects:
           volumeMounts:
           - name: db
             mountPath: /var/lib/heketi
+          - name: config
+            mountPath: /etc/heketi
           readinessProbe:
             timeoutSeconds: 3
             initialDelaySeconds: 3
@@ -98,6 +100,9 @@ objects:
           glusterfs:
             endpoints: heketi-db-${CLUSTER_NAME}-endpoints
             path: heketidbstorage
+        - name: config
+          secret:
+            secretName: heketi-${CLUSTER_NAME}-config-secret
 parameters:
 - name: HEKETI_USER_KEY
   displayName: Heketi User Secret
@@ -105,6 +110,10 @@ parameters:
 - name: HEKETI_ADMIN_KEY
   displayName: Heketi Administrator Secret
   description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_EXECUTOR
+  displayName: heketi executor type
+  description: Set the executor type, kubernetes or ssh
+  value: kubernetes
 - name: HEKETI_KUBE_NAMESPACE
   displayName: Namespace
   description: Set the namespace where the GlusterFS pods reside
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index af901103e..99ad029da 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -19,6 +19,8 @@
     name: "heketi-storage-endpoints"
   - kind: "secret"
     name: "heketi-{{ glusterfs_name }}-topology-secret"
+  - kind: "secret"
+    name: "heketi-{{ glusterfs_name }}-config-secret"
   - kind: "template,route,service,dc"
     name: "heketi-{{ glusterfs_name }}"
   - kind: "svc"
@@ -125,6 +127,13 @@
   when:
   - glusterfs_heketi_topology_load
 
+- name: Generate heketi config file
+  template:
+    src: "{{ openshift.common.examples_content_version }}/heketi.json.j2"
+    dest: "{{ mktemp.stdout }}/heketi.json"
+  when:
+  - glusterfs_heketi_is_native
+
 - name: Generate heketi admin key
   set_fact:
     glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
@@ -142,6 +151,20 @@
   - glusterfs_heketi_is_native
   - glusterfs_heketi_user_key is undefined
 
+- name: Create heketi config secret
+  oc_secret:
+    namespace: "{{ glusterfs_namespace }}"
+    state: present
+    name: "heketi-{{ glusterfs_name }}-config-secret"
+    force: True
+    files:
+    - name: heketi.json
+      path: "{{ mktemp.stdout }}/heketi.json"
+    - name: private_key
+      path: "{{ glusterfs_heketi_ssh_keyfile }}"
+  when:
+  - glusterfs_heketi_is_native
+
 - include: heketi_deploy_part1.yml
   when:
   - glusterfs_heketi_is_native
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
index dbfe126a4..76611d936 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
@@ -20,6 +20,11 @@
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}"
     glusterfs_heketi_port: "{{ openshift_storage_glusterfs_heketi_port }}"
+    glusterfs_heketi_executor: "{{ openshift_storage_glusterfs_heketi_executor }}"
+    glusterfs_heketi_ssh_port: "{{ openshift_storage_glusterfs_heketi_ssh_port }}"
+    glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_heketi_ssh_user }}"
+    glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_heketi_ssh_sudo }}"
+    glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_heketi_ssh_keyfile }}"
     glusterfs_nodes: "{{ groups.glusterfs }}"
 
 - include: glusterfs_common.yml
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
index 0849f2a2e..280088fe4 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -20,6 +20,11 @@
     glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}"
     glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}"
     glusterfs_heketi_port: "{{ openshift_storage_glusterfs_registry_heketi_port }}"
+    glusterfs_heketi_executor: "{{ openshift_storage_glusterfs_registry_heketi_executor }}"
+    glusterfs_heketi_ssh_port: "{{ openshift_storage_glusterfs_registry_heketi_ssh_port }}"
+    glusterfs_heketi_ssh_user: "{{ openshift_storage_glusterfs_registry_heketi_ssh_user }}"
+    glusterfs_heketi_ssh_sudo: "{{ openshift_storage_glusterfs_registry_heketi_ssh_sudo }}"
+    glusterfs_heketi_ssh_keyfile: "{{ openshift_storage_glusterfs_registry_heketi_ssh_keyfile }}"
     glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}"
 
 - include: glusterfs_common.yml
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
index ea9b1fe1f..3ba1eb2d2 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
@@ -36,6 +36,7 @@
       HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
       CLUSTER_NAME: "{{ glusterfs_name }}"
       TOPOLOGY_PATH: "{{ mktemp.stdout }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
index 63009c539..5ef4b5c83 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -106,6 +106,7 @@
       HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}"
       HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
       HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+      HEKETI_EXECUTOR: "{{ glusterfs_heketi_executor }}"
       HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
       CLUSTER_NAME: "{{ glusterfs_name }}"
 
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi.json.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi.json.j2
new file mode 100644
index 000000000..579b11bb7
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi.json.j2
@@ -0,0 +1,36 @@
+{
+	"_port_comment": "Heketi Server Port Number",
+	"port" : "8080",
+
+	"_use_auth": "Enable JWT authorization. Please enable for deployment",
+	"use_auth" : false,
+
+	"_jwt" : "Private keys for access",
+	"jwt" : {
+		"_admin" : "Admin has access to all APIs",
+		"admin" : {
+			"key" : "My Secret"
+		},
+		"_user" : "User only has access to /volumes endpoint",
+		"user" : {
+			"key" : "My Secret"
+		}
+	},
+
+	"_glusterfs_comment": "GlusterFS Configuration",
+	"glusterfs" : {
+
+		"_executor_comment": "Execute plugin. Possible choices: mock, kubernetes, ssh",
+		"executor" : "{{ glusterfs_heketi_executor }}",
+
+		"_db_comment": "Database file name",
+		"db" : "/var/lib/heketi/heketi.db",
+
+		"sshexec" : {
+			"keyfile" : "/etc/heketi/private_key",
+			"port" : "{{ glusterfs_heketi_ssh_port }}",
+			"user" : "{{ glusterfs_heketi_ssh_user }}",
+			"sudo" : {{ glusterfs_heketi_ssh_sudo | lower }}
+		}
+	}
+}