From 73d0a3857a5e623c6f2403ca0575bc28cd048d17 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 10 May 2017 10:12:59 -0500 Subject: GlusterFS: Use groups variables Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml | 2 +- roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml | 6 ++++-- roles/openshift_storage_glusterfs/tasks/main.yml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 451990240..7244b6697 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -17,6 +17,6 @@ glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}" glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}" glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}" - glusterfs_nodes: "{{ g_glusterfs_hosts }}" + glusterfs_nodes: "{{ groups.glusterfs }}" - include: glusterfs_common.yml diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 392f4b65b..7fc312880 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -17,10 +17,12 @@ glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_registry_heketi_topology_load }}" glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}" glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}" - glusterfs_nodes: "{{ g_glusterfs_registry_hosts }}" + glusterfs_nodes: "{{ groups.glusterfs_registry }}" - include: glusterfs_common.yml - when: g_glusterfs_registry_hosts != g_glusterfs_hosts + when: + - "'glusterfs_registry' in groups" + - "'glusterfs' not in groups or groups.glusterfs_registry != groups.glusterfs" - name: Delete pre-existing GlusterFS registry resources oc_obj: diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml index ebd8db453..3efd995c0 100644 --- a/roles/openshift_storage_glusterfs/tasks/main.yml +++ b/roles/openshift_storage_glusterfs/tasks/main.yml @@ -7,11 +7,11 @@ - include: glusterfs_config.yml when: - - g_glusterfs_hosts | default([]) | count > 0 + - groups.glusterfs | default([]) | count > 0 - include: glusterfs_registry.yml when: - - g_glusterfs_registry_hosts | default([]) | count > 0 + - groups.glusterfs_registry | default([]) | count > 0 - "openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap" - name: Delete temp directory -- cgit v1.2.3 From 06f01c77d4a2901d637201ded3db3ff8de6681c6 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Thu, 11 May 2017 19:09:43 -0500 Subject: GlusterFS: Make sure timeout is an int Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml | 6 +++--- roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml | 2 +- roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml | 2 +- roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index fa5fa2cb0..4c68e746b 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -39,7 +39,7 @@ register: heketi_pod until: "heketi_pod.results.results[0]['items'] | count == 0" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" when: glusterfs_heketi_wipe - name: Wait for heketi pods to terminate @@ -51,7 +51,7 @@ register: heketi_pod until: "heketi_pod.results.results[0]['items'] | count == 0" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" when: glusterfs_heketi_wipe - name: Create heketi service account @@ -130,7 +130,7 @@ - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" when: - glusterfs_heketi_is_native - glusterfs_heketi_url is undefined diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index 579112349..782978710 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -104,4 +104,4 @@ # There must be as many pods with 'Ready' staus True as there are nodes expecting those pods - "glusterfs_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == glusterfs_nodes | count" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index c14fcfb15..f1975b59f 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -38,4 +38,4 @@ # Pod's 'Ready' status must be True - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index 64410a9ab..e060fcd66 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -28,7 +28,7 @@ # Pod's 'Complete' status must be True - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Complete'}) | map('bool') | select | list | count == 1" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" failed_when: - "'results' in heketi_job.results" - "heketi_job.results.results | count > 0" @@ -85,7 +85,7 @@ # Pod's 'Ready' status must be True - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" - name: Determine heketi URL oc_obj: @@ -98,7 +98,7 @@ - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''" delay: 10 - retries: "{{ (glusterfs_timeout / 10) | int }}" + retries: "{{ (glusterfs_timeout | int / 10) | int }}" - name: Set heketi URL set_fact: -- cgit v1.2.3 From fb96b04bdb1b21562bec5a17dda753f93a4a7877 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Thu, 11 May 2017 21:46:50 -0500 Subject: GlusterFS: Label on Openshift node name Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index 782978710..1b3814b0b 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -61,13 +61,11 @@ - name: Label GlusterFS nodes oc_label: - name: "{{ glusterfs_host }}" + name: "{{ hostvars[item].openshift.common.hostname }}" kind: node state: add labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}" with_items: "{{ glusterfs_nodes | default([]) }}" - loop_control: - loop_var: glusterfs_host - name: Copy GlusterFS DaemonSet template copy: -- cgit v1.2.3 From 378a80feea73cd1964e2e49a43380cc733e281a0 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Fri, 12 May 2017 11:02:51 -0500 Subject: GlusterFS: Allow for configuration of node selector Signed-off-by: Jose A. Rivera --- .../openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml | 7 +++++-- roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml index c66705752..ab4ca3d37 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml @@ -28,8 +28,7 @@ objects: labels: glusterfs-node: pod spec: - nodeSelector: - storagenode: glusterfs + nodeSelector: "${{NODE_LABELS}}" hostNetwork: true containers: - name: glusterfs @@ -120,6 +119,10 @@ objects: dnsPolicy: ClusterFirst securityContext: {} parameters: +- name: NODE_LABELS + displayName: Daemonset Node Labels + description: Labels which define the daemonset node selector. + value: '{ "storagenode": "glusterfs" }' - name: IMAGE_NAME displayName: GlusterFS container name required: True diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index 1b3814b0b..bd7987e30 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -89,6 +89,7 @@ params: IMAGE_NAME: "{{ glusterfs_image }}" IMAGE_VERSION: "{{ glusterfs_version }}" + NODE_LABELS: "{{ glusterfs_nodeselector }}" - name: Wait for GlusterFS pods oc_obj: -- cgit v1.2.3 From f4093a395dc391a80a402d0b0384c94934d0812c Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Fri, 12 May 2017 11:03:42 -0500 Subject: GlusterFS: Tweak pod probe parameters Signed-off-by: Jose A. Rivera --- .../files/v3.6/glusterfs-template.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml index ab4ca3d37..66f2f1c55 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml @@ -62,26 +62,26 @@ objects: privileged: true readinessProbe: timeoutSeconds: 3 - initialDelaySeconds: 100 + initialDelaySeconds: 40 exec: command: - "/bin/bash" - "-c" - systemctl status glusterd.service - periodSeconds: 10 + periodSeconds: 25 successThreshold: 1 - failureThreshold: 3 + failureThreshold: 15 livenessProbe: timeoutSeconds: 3 - initialDelaySeconds: 100 + initialDelaySeconds: 40 exec: command: - "/bin/bash" - "-c" - systemctl status glusterd.service - periodSeconds: 10 + periodSeconds: 25 successThreshold: 1 - failureThreshold: 3 + failureThreshold: 15 resources: {} terminationMessagePath: "/dev/termination-log" volumes: -- cgit v1.2.3 From 0cdd84e54406c41033ad8cec9e88fc425bae572c Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Fri, 12 May 2017 12:21:58 -0500 Subject: GlusterFS: Allow failure reporting when deleting deploy-heketi Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index e060fcd66..d58053a1a 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -46,7 +46,6 @@ with_items: - kind: "template,route,service,jobs,dc,secret" selector: "deploy-heketi" - failed_when: False - name: Copy heketi template copy: -- cgit v1.2.3 From 9c06b1e7fc9cdf7a5cc190ce7f683c00b60d97e8 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Mon, 15 May 2017 15:55:00 -0500 Subject: GlusterFS: Adjust when clauses for registry config Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml | 2 +- roles/openshift_storage_glusterfs/tasks/main.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 7fc312880..2d06a2d24 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -21,7 +21,7 @@ - include: glusterfs_common.yml when: - - "'glusterfs_registry' in groups" + - groups.glusterfs_registry | default([]) | count > 0 - "'glusterfs' not in groups or groups.glusterfs_registry != groups.glusterfs" - name: Delete pre-existing GlusterFS registry resources diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml index 3efd995c0..c9bfdd1cd 100644 --- a/roles/openshift_storage_glusterfs/tasks/main.yml +++ b/roles/openshift_storage_glusterfs/tasks/main.yml @@ -11,8 +11,7 @@ - include: glusterfs_registry.yml when: - - groups.glusterfs_registry | default([]) | count > 0 - - "openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap" + - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap" - name: Delete temp directory file: -- cgit v1.2.3 From efc6b81fa415cb1498ca0256cc27f3fb9d72ace7 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Thu, 18 May 2017 14:34:41 -0500 Subject: GlusterFS: Allow configuration of kube namespace for heketi Signed-off-by: Jose A. Rivera --- .../files/v3.6/deploy-heketi-template.yml | 6 ++++++ roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml | 6 ++++++ roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml | 1 + roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml | 1 + 4 files changed, 14 insertions(+) diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml index c9945be13..c132e9c2c 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml @@ -81,6 +81,8 @@ objects: value: '14' - name: HEKETI_KUBE_GLUSTER_DAEMONSET value: '1' + - name: HEKETI_KUBE_NAMESPACE + value: ${HEKETI_KUBE_NAMESPACE} ports: - containerPort: 8080 volumeMounts: @@ -107,6 +109,10 @@ parameters: - name: HEKETI_ADMIN_KEY displayName: Heketi Administrator Secret description: Set secret for administration of the Heketi service as user _admin_ +- name: HEKETI_KUBE_NAMESPACE + displayName: Namespace + description: Set the namespace where the GlusterFS pods reside + value: default - name: IMAGE_NAME displayName: GlusterFS container name required: True diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml index df045c170..e9b9b782e 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml @@ -76,6 +76,8 @@ objects: value: '14' - name: HEKETI_KUBE_GLUSTER_DAEMONSET value: '1' + - name: HEKETI_KUBE_NAMESPACE + value: ${HEKETI_KUBE_NAMESPACE} ports: - containerPort: 8080 volumeMounts: @@ -105,6 +107,10 @@ parameters: - name: HEKETI_ADMIN_KEY displayName: Heketi Administrator Secret description: Set secret for administration of the Heketi service as user _admin_ +- name: HEKETI_KUBE_NAMESPACE + displayName: Namespace + description: Set the namespace where the GlusterFS pods reside + value: default - name: IMAGE_NAME displayName: GlusterFS container name required: True diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index f1975b59f..91abd8156 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -25,6 +25,7 @@ IMAGE_VERSION: "{{ glusterfs_heketi_version }}" HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" + HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" - name: Wait for deploy-heketi pod oc_obj: diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index d58053a1a..d4fbba97a 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -71,6 +71,7 @@ IMAGE_VERSION: "{{ glusterfs_heketi_version }}" HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" + HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" - name: Wait for heketi pod oc_obj: -- cgit v1.2.3 From 06604bdd4de165c9ef97ad64f14374b18f9e2d13 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Mon, 22 May 2017 14:24:32 -0500 Subject: GlusterFS: Improve documentation Signed-off-by: Jose A. Rivera --- inventory/byo/hosts.byo.native-glusterfs.example | 2 +- roles/openshift_storage_glusterfs/README.md | 61 ++++++++++++++++++++---- 2 files changed, 54 insertions(+), 9 deletions(-) diff --git a/inventory/byo/hosts.byo.native-glusterfs.example b/inventory/byo/hosts.byo.native-glusterfs.example index 2dbb57d40..dc847a5b2 100644 --- a/inventory/byo/hosts.byo.native-glusterfs.example +++ b/inventory/byo/hosts.byo.native-glusterfs.example @@ -24,7 +24,7 @@ glusterfs [OSEv3:vars] ansible_ssh_user=root -deployment_type=origin +openshift_deployment_type=origin # Specify that we want to use GlusterFS storage for a hosted registry openshift_hosted_registry_storage_kind=glusterfs diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md index 7b310dbf8..2e2b2497b 100644 --- a/roles/openshift_storage_glusterfs/README.md +++ b/roles/openshift_storage_glusterfs/README.md @@ -1,7 +1,26 @@ OpenShift GlusterFS Cluster =========================== -OpenShift GlusterFS Cluster Installation +OpenShift GlusterFS Cluster Configuration + +This role handles the configuration of GlusterFS clusters. It can handle +two primary configuration scenarios: + +* Configuring a new, natively-hosted GlusterFS cluster. In this scenario, + GlusterFS pods are deployed on nodes in the OpenShift cluster which are + configured to provide storage. +* Configuring a new, external GlusterFS cluster. In this scenario, the + cluster nodes have the GlusterFS software pre-installed but have not + been configured yet. The installer will take care of configuring the + cluster(s) for use by OpenShift applications. +* Using existing GlusterFS clusters. In this scenario, one or more + GlusterFS clusters are assumed to be already setup. These clusters can + be either natively-hosted or external, but must be managed by a + [heketi service](https://github.com/heketi/heketi). + +As part of the configuration, a particular GlusterFS cluster may be +specified to provide backend storage for a natively-hosted Docker +registry. Requirements ------------ @@ -21,13 +40,35 @@ hosted Docker registry: * `[glusterfs_registry]` +Host Variables +-------------- + +For configuring new clusters, the following role variables are available. + +Each host in either of the above groups must have the following variable +defined: + +| Name | Default value | Description | +|-------------------|---------------|-----------------------------------------| +| glusterfs_devices | None | A list of block devices that will be completely managed as part of a GlusterFS cluster. There must be at least one device listed. Each device must be bare, e.g. no partitions or LVM PVs. **Example:** '[ "/dev/sdb" ]' + +In addition, each host may specify the following variables to further control +their configuration as GlusterFS nodes: + +| Name | Default value | Description | +|--------------------|---------------------------|-----------------------------------------| +| glusterfs_cluster | 1 | The ID of the cluster this node should belong to. This is useful when a single heketi service is expected to manage multiple distinct clusters. **NOTE:** For natively-hosted clusters, all pods will be in the same OpenShift namespace +| glusterfs_hostname | openshift.common.hostname | A hostname (or IP address) that will be used for internal GlusterFS communication +| glusterfs_ip | openshift.common.ip | An IP address that will be used by pods to communicate with the GlusterFS node +| glusterfs_zone | 1 | A zone number for the node. Zones are used within the cluster for determining how to distribute the bricks of GlusterFS volumes. heketi will try to spread each volumes' bricks as evenly as possible across all zones + Role Variables -------------- This role has the following variables that control the integration of a GlusterFS cluster into a new or existing OpenShift cluster: -| Name | Default value | | +| Name | Default value | Description | |--------------------------------------------------|-------------------------|-----------------------------------------| | openshift_storage_glusterfs_timeout | 300 | Seconds to wait for pods to become ready | openshift_storage_glusterfs_namespace | 'default' | Namespace in which to create GlusterFS resources @@ -52,17 +93,21 @@ registry. These variables start with the prefix values in their corresponding non-registry variables. The following variables are an exception: -| Name | Default value | | +| Name | Default value | Description | |---------------------------------------------------|-----------------------|-----------------------------------------| | openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' | openshift_storage_glusterfs_registry_nodeselector | 'storagenode=registry'| This allows for the logical separation of the registry GlusterFS cluster from any regular-use GlusterFS clusters Additionally, this role's behavior responds to the following registry-specific -variable: - -| Name | Default value | Description | -|----------------------------------------------|---------------|------------------------------------------------------------------------------| -| openshift_hosted_registry_glusterfs_swap | False | Whether to swap an existing registry's storage volume for a GlusterFS volume | +variables: + +| Name | Default value | Description | +|-----------------------------------------------|------------------------------|-----------------------------------------| +| openshift_hosted_registry_glusterfs_endpoints | glusterfs-registry-endpoints | The name for the Endpoints resource that will point the registry to the GlusterFS nodes +| openshift_hosted_registry_glusterfs_path | glusterfs-registry-volume | The name for the GlusterFS volume that will provide registry storage +| openshift_hosted_registry_glusterfs_readonly | False | Whether the GlusterFS volume should be read-only +| openshift_hosted_registry_glusterfs_swap | False | Whether to swap an existing registry's storage volume for a GlusterFS volume +| openshift_hosted_registry_glusterfs_swapcopy | True | If swapping, copy the contents of the pre-existing registry storage to the new GlusterFS volume Dependencies ------------ -- cgit v1.2.3 From 5e73aa7b07fefbb211c791f4b5b5f8d29af7860f Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 31 May 2017 10:36:41 -0500 Subject: GlusterFS: Minor corrections and cleanups Signed-off-by: Jose A. Rivera --- .../files/v3.6/deploy-heketi-template.yml | 8 +++----- roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml | 6 ++---- roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml | 6 +----- roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml | 3 +-- roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml | 4 ++-- roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml | 2 +- 6 files changed, 10 insertions(+), 19 deletions(-) diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml index c132e9c2c..8523f016b 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml @@ -9,8 +9,6 @@ metadata: annotations: description: Bootstrap Heketi installation tags: glusterfs,heketi,installation -labels: - template: deploy-heketi objects: - kind: Service apiVersion: v1 @@ -66,7 +64,7 @@ objects: spec: serviceAccountName: heketi-service-account containers: - - name: deploy-heketi + - name: heketi image: ${IMAGE_NAME}:${IMAGE_VERSION} env: - name: HEKETI_USER_KEY @@ -114,8 +112,8 @@ parameters: description: Set the namespace where the GlusterFS pods reside value: default - name: IMAGE_NAME - displayName: GlusterFS container name + displayName: heketi container name required: True - name: IMAGE_VERSION - displayName: GlusterFS container versiona + displayName: heketi container versiona required: True diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml index e9b9b782e..82cf726ac 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml @@ -8,8 +8,6 @@ metadata: annotations: description: Heketi service deployment template tags: glusterfs,heketi -labels: - template: heketi objects: - kind: Service apiVersion: v1 @@ -112,8 +110,8 @@ parameters: description: Set the namespace where the GlusterFS pods reside value: default - name: IMAGE_NAME - displayName: GlusterFS container name + displayName: heketi container name required: True - name: IMAGE_VERSION - displayName: GlusterFS container versiona + displayName: heketi container versiona required: True diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index bd7987e30..d0458bf93 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -1,8 +1,4 @@ --- -- assert: - that: "glusterfs_nodeselector.keys() | count == 1" - msg: Only one GlusterFS nodeselector key pair should be provided - - assert: that: "glusterfs_nodes | count >= 3" msg: There must be at least three GlusterFS nodes specified @@ -17,7 +13,7 @@ - name: Unlabel any existing GlusterFS nodes oc_label: - name: "{{ item }}" + name: "{{ hostvars[item].openshift.common.hostname }}" kind: node state: absent labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 2d06a2d24..c7419d5b3 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -28,8 +28,7 @@ oc_obj: namespace: "{{ glusterfs_namespace }}" kind: "{{ item.kind }}" - name: "{{ item.name | default(omit) }}" - selector: "{{ item.selector | default(omit) }}" + name: "{{ item.name }}" state: absent with_items: - kind: "svc,ep" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index 91abd8156..c7bcd0e35 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -6,11 +6,11 @@ with_items: - "deploy-heketi-template.yml" -- name: Create deploy-heketi resources +- name: Create deploy-heketi template oc_obj: namespace: "{{ glusterfs_namespace }}" kind: template - name: deploy-heketi + name: "deploy-heketi" state: present files: - "{{ mktemp.stdout }}/deploy-heketi-template.yml" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index d4fbba97a..127b7d618 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -52,7 +52,7 @@ src: "{{ openshift.common.examples_content_version }}/heketi-template.yml" dest: "{{ mktemp.stdout }}/heketi-template.yml" -- name: Create heketi resources +- name: Create heketi template oc_obj: namespace: "{{ glusterfs_namespace }}" kind: template -- cgit v1.2.3 From a315411dee1217518bbd83bf6508bc44e0fec5d6 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 31 May 2017 11:48:33 -0500 Subject: GlusterFS: Allow cleaner separation of multiple clusters Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/README.md | 5 ++-- .../openshift_storage_glusterfs/defaults/main.yml | 6 ++-- .../files/v3.6/deploy-heketi-template.yml | 28 +++++++++-------- .../files/v3.6/glusterfs-registry-service.yml | 10 ------- .../files/v3.6/glusterfs-template.yml | 16 ++++++---- .../files/v3.6/heketi-template.yml | 29 ++++++++++-------- .../tasks/glusterfs_common.yml | 28 +++++++++-------- .../tasks/glusterfs_config.yml | 3 +- .../tasks/glusterfs_deploy.yml | 14 ++++++--- .../tasks/glusterfs_registry.yml | 15 +++++----- .../tasks/heketi_deploy_part1.yml | 3 +- .../tasks/heketi_deploy_part2.yml | 35 ++++++++++++++++++++-- .../v3.6/glusterfs-registry-endpoints.yml.j2 | 3 +- .../v3.6/glusterfs-registry-service.yml.j2 | 10 +++++++ .../templates/v3.6/heketi-endpoints.yml.j2 | 12 ++++++++ .../templates/v3.6/heketi-service.yml.j2 | 10 +++++++ 16 files changed, 153 insertions(+), 74 deletions(-) delete mode 100644 roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml create mode 100644 roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 create mode 100644 roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 create mode 100644 roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md index 2e2b2497b..59af7523f 100644 --- a/roles/openshift_storage_glusterfs/README.md +++ b/roles/openshift_storage_glusterfs/README.md @@ -73,7 +73,8 @@ GlusterFS cluster into a new or existing OpenShift cluster: | openshift_storage_glusterfs_timeout | 300 | Seconds to wait for pods to become ready | openshift_storage_glusterfs_namespace | 'default' | Namespace in which to create GlusterFS resources | openshift_storage_glusterfs_is_native | True | GlusterFS should be containerized -| openshift_storage_glusterfs_nodeselector | 'storagenode=glusterfs' | Selector to determine which nodes will host GlusterFS pods in native mode +| openshift_storage_glusterfs_name | 'storage' | A name to identify the GlusterFS cluster, which will be used in resource names +| openshift_storage_glusterfs_nodeselector | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name | openshift_storage_glusterfs_image | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7' | openshift_storage_glusterfs_version | 'latest' | Container image version to use for GlusterFS pods | openshift_storage_glusterfs_wipe | False | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.** @@ -96,7 +97,7 @@ are an exception: | Name | Default value | Description | |---------------------------------------------------|-----------------------|-----------------------------------------| | openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' -| openshift_storage_glusterfs_registry_nodeselector | 'storagenode=registry'| This allows for the logical separation of the registry GlusterFS cluster from any regular-use GlusterFS clusters +| openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters Additionally, this role's behavior responds to the following registry-specific variables: diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml index ebe9ca30b..ba4db8357 100644 --- a/roles/openshift_storage_glusterfs/defaults/main.yml +++ b/roles/openshift_storage_glusterfs/defaults/main.yml @@ -2,7 +2,8 @@ openshift_storage_glusterfs_timeout: 300 openshift_storage_glusterfs_namespace: 'default' openshift_storage_glusterfs_is_native: True -openshift_storage_glusterfs_nodeselector: 'storagenode=glusterfs' +openshift_storage_glusterfs_name: 'storage' +openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host" openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}" openshift_storage_glusterfs_version: 'latest' openshift_storage_glusterfs_wipe: False @@ -20,7 +21,8 @@ openshift_storage_glusterfs_heketi_url: "{{ omit }}" openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}" openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}" openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}" -openshift_storage_glusterfs_registry_nodeselector: 'storagenode=registry' +openshift_storage_glusterfs_registry_name: 'registry' +openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host" openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}" openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}" openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}" diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml index 8523f016b..c67e05254 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml @@ -13,43 +13,43 @@ objects: - kind: Service apiVersion: v1 metadata: - name: deploy-heketi + name: deploy-heketi-${CLUSTER_NAME} labels: - glusterfs: deploy-heketi-service + glusterfs: deploy-heketi-${CLUSTER_NAME}-service deploy-heketi: support annotations: description: Exposes Heketi service spec: ports: - - name: deploy-heketi + - name: deploy-heketi-${CLUSTER_NAME} port: 8080 targetPort: 8080 selector: - name: deploy-heketi + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod - kind: Route apiVersion: v1 metadata: - name: deploy-heketi + name: deploy-heketi-${CLUSTER_NAME} labels: - glusterfs: deploy-heketi-route + glusterfs: deploy-heketi-${CLUSTER_NAME}-route deploy-heketi: support spec: to: kind: Service - name: deploy-heketi + name: deploy-heketi-${CLUSTER_NAME} - kind: DeploymentConfig apiVersion: v1 metadata: - name: deploy-heketi + name: deploy-heketi-${CLUSTER_NAME} labels: - glusterfs: deploy-heketi-dc + glusterfs: deploy-heketi-${CLUSTER_NAME}-dc deploy-heketi: support annotations: description: Defines how to deploy Heketi spec: replicas: 1 selector: - name: deploy-heketi + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod triggers: - type: ConfigChange strategy: @@ -58,11 +58,10 @@ objects: metadata: name: deploy-heketi labels: - name: deploy-heketi - glusterfs: deploy-heketi-pod + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod deploy-heketi: support spec: - serviceAccountName: heketi-service-account + serviceAccountName: heketi-${CLUSTER_NAME}-service-account containers: - name: heketi image: ${IMAGE_NAME}:${IMAGE_VERSION} @@ -117,3 +116,6 @@ parameters: - name: IMAGE_VERSION displayName: heketi container versiona required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + value: glusterfs diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml deleted file mode 100644 index 3f8d8f507..000000000 --- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: v1 -kind: Service -metadata: - name: glusterfs-registry-endpoints -spec: - ports: - - port: 1 -status: - loadBalancer: {} diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml index 66f2f1c55..dc3d2250a 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml @@ -12,20 +12,21 @@ objects: - kind: DaemonSet apiVersion: extensions/v1beta1 metadata: - name: glusterfs + name: glusterfs-${CLUSTER_NAME} labels: - glusterfs: daemonset + glusterfs: ${CLUSTER_NAME}-daemonset annotations: description: GlusterFS DaemonSet tags: glusterfs spec: selector: matchLabels: - glusterfs-node: pod + glusterfs: ${CLUSTER_NAME}-pod template: metadata: - name: glusterfs + name: glusterfs-${CLUSTER_NAME} labels: + glusterfs: ${CLUSTER_NAME}-pod glusterfs-node: pod spec: nodeSelector: "${{NODE_LABELS}}" @@ -121,11 +122,14 @@ objects: parameters: - name: NODE_LABELS displayName: Daemonset Node Labels - description: Labels which define the daemonset node selector. - value: '{ "storagenode": "glusterfs" }' + description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=-host\' + value: '{ "glusterfs": "storage-host" }' - name: IMAGE_NAME displayName: GlusterFS container name required: True - name: IMAGE_VERSION displayName: GlusterFS container versiona required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + value: storage diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml index 82cf726ac..1d8f1abdf 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml @@ -12,9 +12,9 @@ objects: - kind: Service apiVersion: v1 metadata: - name: heketi + name: heketi-${CLUSTER_NAME} labels: - glusterfs: heketi-service + glusterfs: heketi-${CLUSTER_NAME}-service annotations: description: Exposes Heketi service spec: @@ -23,40 +23,40 @@ objects: port: 8080 targetPort: 8080 selector: - glusterfs: heketi-pod + glusterfs: heketi-${CLUSTER_NAME}-pod - kind: Route apiVersion: v1 metadata: - name: heketi + name: heketi-${CLUSTER_NAME} labels: - glusterfs: heketi-route + glusterfs: heketi-${CLUSTER_NAME}-route spec: to: kind: Service - name: heketi + name: heketi-${CLUSTER_NAME} - kind: DeploymentConfig apiVersion: v1 metadata: - name: heketi + name: heketi-${CLUSTER_NAME} labels: - glusterfs: heketi-dc + glusterfs: heketi-${CLUSTER_NAME}-dc annotations: description: Defines how to deploy Heketi spec: replicas: 1 selector: - glusterfs: heketi-pod + glusterfs: heketi-${CLUSTER_NAME}-pod triggers: - type: ConfigChange strategy: type: Recreate template: metadata: - name: heketi + name: heketi-${CLUSTER_NAME} labels: - glusterfs: heketi-pod + glusterfs: heketi-${CLUSTER_NAME}-pod spec: - serviceAccountName: heketi-service-account + serviceAccountName: heketi-${CLUSTER_NAME}-service-account containers: - name: heketi image: ${IMAGE_NAME}:${IMAGE_VERSION} @@ -96,7 +96,7 @@ objects: volumes: - name: db glusterfs: - endpoints: heketi-storage-endpoints + endpoints: heketi-db-${CLUSTER_NAME}-endpoints path: heketidbstorage parameters: - name: HEKETI_USER_KEY @@ -115,3 +115,6 @@ parameters: - name: IMAGE_VERSION displayName: heketi container versiona required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + value: glusterfs diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 4c68e746b..cb0fc0fbb 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -21,12 +21,16 @@ with_items: - kind: "template,route,service,dc,jobs,secret" selector: "deploy-heketi" - - kind: "template,route,service,dc" - name: "heketi" - - kind: "svc,ep" + - kind: "svc" name: "heketi-storage-endpoints" + - kind: "template,route,service,dc" + name: "heketi-{{ glusterfs_name }}" + - kind: "svc" + name: "heketi-db-{{ glusterfs_name }}-endpoints" - kind: "sa" - name: "heketi-service-account" + name: "heketi-{{ glusterfs_name }}-service-account" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-secret" failed_when: False when: glusterfs_heketi_wipe @@ -35,7 +39,7 @@ namespace: "{{ glusterfs_namespace }}" kind: pod state: list - selector: "glusterfs=deploy-heketi-pod" + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" register: heketi_pod until: "heketi_pod.results.results[0]['items'] | count == 0" delay: 10 @@ -47,7 +51,7 @@ namespace: "{{ glusterfs_namespace }}" kind: pod state: list - selector: "glusterfs=heketi-pod" + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" register: heketi_pod until: "heketi_pod.results.results[0]['items'] | count == 0" delay: 10 @@ -57,13 +61,13 @@ - name: Create heketi service account oc_serviceaccount: namespace: "{{ glusterfs_namespace }}" - name: heketi-service-account + name: "heketi-{{ glusterfs_name }}-service-account" state: present when: glusterfs_heketi_is_native - name: Add heketi service account to privileged SCC oc_adm_policy_user: - user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account" + user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account" resource_kind: scc resource_name: privileged state: present @@ -71,7 +75,7 @@ - name: Allow heketi service account to view/edit pods oc_adm_policy_user: - user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account" + user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account" resource_kind: role resource_name: edit state: present @@ -82,7 +86,7 @@ namespace: "{{ glusterfs_namespace }}" state: list kind: pod - selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support" + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" register: heketi_pod when: glusterfs_heketi_is_native @@ -100,7 +104,7 @@ namespace: "{{ glusterfs_namespace }}" state: list kind: pod - selector: "glusterfs=heketi-pod" + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" register: heketi_pod when: glusterfs_heketi_is_native @@ -124,7 +128,7 @@ namespace: "{{ glusterfs_namespace }}" state: list kind: ep - selector: "glusterfs in (deploy-heketi-service, heketi-service)" + selector: "glusterfs in (deploy-heketi-{{ glusterfs_name }}-service, heketi-{{ glusterfs_name }}-service)" register: heketi_url until: - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 7244b6697..4c693528f 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -3,7 +3,8 @@ glusterfs_timeout: "{{ openshift_storage_glusterfs_timeout }}" glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}" glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}" - glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | map_from_pairs }}" + glusterfs_name: "{{ openshift_storage_glusterfs_name }}" + glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}" glusterfs_image: "{{ openshift_storage_glusterfs_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_version }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index d0458bf93..bb831147d 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -6,9 +6,14 @@ - name: Delete pre-existing GlusterFS resources oc_obj: namespace: "{{ glusterfs_namespace }}" - kind: "template,daemonset" - name: glusterfs + kind: "{{ item.kind }}" + name: "{{ item.name }}" state: absent + with_items: + - kind: template + name: glusterfs + - kind: daemonset + name: "glusterfs-{{ glusterfs_name }}" when: glusterfs_wipe - name: Unlabel any existing GlusterFS nodes @@ -72,7 +77,7 @@ oc_obj: namespace: "{{ glusterfs_namespace }}" kind: template - name: glusterfs + name: "glusterfs" state: present files: - "{{ mktemp.stdout }}/glusterfs-template.yml" @@ -86,13 +91,14 @@ IMAGE_NAME: "{{ glusterfs_image }}" IMAGE_VERSION: "{{ glusterfs_version }}" NODE_LABELS: "{{ glusterfs_nodeselector }}" + CLUSTER_NAME: "{{ glusterfs_name }}" - name: Wait for GlusterFS pods oc_obj: namespace: "{{ glusterfs_namespace }}" kind: pod state: list - selector: "glusterfs-node=pod" + selector: "glusterfs={{ glusterfs_name }}-pod" register: glusterfs_pods until: - "glusterfs_pods.results.results[0]['items'] | count > 0" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index c7419d5b3..7bfa0436c 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -3,7 +3,8 @@ glusterfs_timeout: "{{ openshift_storage_glusterfs_registry_timeout }}" glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}" glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}" - glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | map_from_pairs }}" + glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}" + glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}" glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}" @@ -31,8 +32,8 @@ name: "{{ item.name }}" state: absent with_items: - - kind: "svc,ep" - name: "glusterfs-registry-endpoints" + - kind: "svc" + name: "glusterfs-{{ glusterfs_name }}-endpoints" failed_when: False - name: Generate GlusterFS registry endpoints @@ -41,8 +42,8 @@ dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml" - name: Copy GlusterFS registry service - copy: - src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml" + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2" dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml" - name: Create GlusterFS registry endpoints @@ -50,7 +51,7 @@ namespace: "{{ glusterfs_namespace }}" state: present kind: endpoints - name: glusterfs-registry-endpoints + name: "glusterfs-{{ glusterfs_name }}-endpoints" files: - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml" @@ -59,7 +60,7 @@ namespace: "{{ glusterfs_namespace }}" state: present kind: service - name: glusterfs-registry-endpoints + name: "glusterfs-{{ glusterfs_name }}-endpoints" files: - "{{ mktemp.stdout }}/glusterfs-registry-service.yml" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index c7bcd0e35..fc4fbe7e1 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -26,13 +26,14 @@ HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + CLUSTER_NAME: "{{ glusterfs_name }}" - name: Wait for deploy-heketi pod oc_obj: namespace: "{{ glusterfs_namespace }}" kind: pod state: list - selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support" + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" register: heketi_pod until: - "heketi_pod.results.results[0]['items'] | count > 0" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index 127b7d618..80aa1f2db 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -46,6 +46,36 @@ with_items: - kind: "template,route,service,jobs,dc,secret" selector: "deploy-heketi" + - kind: "svc" + name: "heketi-storage-endpoints" + +- name: Generate heketi endpoints + template: + src: "{{ openshift.common.examples_content_version }}/heketi-endpoints.yml.j2" + dest: "{{ mktemp.stdout }}/heketi-endpoints.yml" + +- name: Generate heketi service + template: + src: "{{ openshift.common.examples_content_version }}/heketi-service.yml.j2" + dest: "{{ mktemp.stdout }}/heketi-service.yml" + +- name: Create heketi endpoints + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: endpoints + name: "heketi-db-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/heketi-endpoints.yml" + +- name: Create heketi service + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: service + name: "heketi-db-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/heketi-service.yml" - name: Copy heketi template copy: @@ -72,13 +102,14 @@ HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + CLUSTER_NAME: "{{ glusterfs_name }}" - name: Wait for heketi pod oc_obj: namespace: "{{ glusterfs_namespace }}" kind: pod state: list - selector: "glusterfs=heketi-pod" + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" register: heketi_pod until: - "heketi_pod.results.results[0]['items'] | count > 0" @@ -92,7 +123,7 @@ namespace: "{{ glusterfs_namespace }}" state: list kind: ep - selector: "glusterfs=heketi-service" + selector: "glusterfs=heketi-{{ glusterfs_name }}-service" register: heketi_url until: - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 index 605627ab5..11c9195bb 100644 --- a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 @@ -1,7 +1,8 @@ +--- apiVersion: v1 kind: Endpoints metadata: - name: glusterfs-registry-endpoints + name: glusterfs-{{ glusterfs_name }}-endpoints subsets: - addresses: {% for node in glusterfs_nodes %} diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 new file mode 100644 index 000000000..3f869d2b7 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: glusterfs-{{ glusterfs_name }}-endpoints +spec: + ports: + - port: 1 +status: + loadBalancer: {} diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 new file mode 100644 index 000000000..99cbdf748 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: heketi-db-{{ glusterfs_name }}-endpoints +subsets: +- addresses: +{% for node in glusterfs_nodes %} + - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }} +{% endfor %} + ports: + - port: 1 diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 new file mode 100644 index 000000000..dcb896441 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: heketi-db-{{ glusterfs_name }}-endpoints +spec: + ports: + - port: 1 +status: + loadBalancer: {} -- cgit v1.2.3 From 81c8ddc5bb409780cf084aeba8877c67970a4af6 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 31 May 2017 12:03:26 -0500 Subject: GlusterFS: Use proper secrets Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/README.md | 14 ++++++++------ roles/openshift_storage_glusterfs/defaults/main.yml | 8 ++++---- .../openshift_storage_glusterfs/tasks/glusterfs_common.yml | 13 ++++++++++++- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md index 59af7523f..5783bcbd9 100644 --- a/roles/openshift_storage_glusterfs/README.md +++ b/roles/openshift_storage_glusterfs/README.md @@ -81,8 +81,8 @@ GlusterFS cluster into a new or existing OpenShift cluster: | openshift_storage_glusterfs_heketi_is_native | True | heketi should be containerized | openshift_storage_glusterfs_heketi_image | 'heketi/heketi' | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7' | openshift_storage_glusterfs_heketi_version | 'latest' | Container image version to use for heketi pods -| openshift_storage_glusterfs_heketi_admin_key | '' | String to use as secret key for performing heketi commands as admin -| openshift_storage_glusterfs_heketi_user_key | '' | String to use as secret key for performing heketi commands as user that can only view or modify volumes +| openshift_storage_glusterfs_heketi_admin_key | auto-generated | String to use as secret key for performing heketi commands as admin +| openshift_storage_glusterfs_heketi_user_key | auto-generated | String to use as secret key for performing heketi commands as user that can only view or modify volumes | openshift_storage_glusterfs_heketi_topology_load | True | Load the GlusterFS topology information into heketi | openshift_storage_glusterfs_heketi_url | Undefined | URL for the heketi REST API, dynamically determined in native mode | openshift_storage_glusterfs_heketi_wipe | False | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe` @@ -94,10 +94,12 @@ registry. These variables start with the prefix values in their corresponding non-registry variables. The following variables are an exception: -| Name | Default value | Description | -|---------------------------------------------------|-----------------------|-----------------------------------------| -| openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' -| openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters +| Name | Default value | Description | +|-------------------------------------------------------|-----------------------|-----------------------------------------| +| openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' +| openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters +| openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated | Separate from the above +| openshift_storage_glusterfs_registry_heketi_user_key | auto-generated | Separate from the above Additionally, this role's behavior responds to the following registry-specific variables: diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml index ba4db8357..b6d06c086 100644 --- a/roles/openshift_storage_glusterfs/defaults/main.yml +++ b/roles/openshift_storage_glusterfs/defaults/main.yml @@ -12,8 +12,8 @@ openshift_storage_glusterfs_heketi_is_missing: True openshift_storage_glusterfs_heketi_deploy_is_missing: True openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}" openshift_storage_glusterfs_heketi_version: 'latest' -openshift_storage_glusterfs_heketi_admin_key: '' -openshift_storage_glusterfs_heketi_user_key: '' +openshift_storage_glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}" +openshift_storage_glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}" openshift_storage_glusterfs_heketi_topology_load: True openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}" openshift_storage_glusterfs_heketi_url: "{{ omit }}" @@ -31,8 +31,8 @@ openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_gl openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}" openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}" openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}" -openshift_storage_glusterfs_registry_heketi_admin_key: "{{ openshift_storage_glusterfs_heketi_admin_key }}" -openshift_storage_glusterfs_registry_heketi_user_key: "{{ openshift_storage_glusterfs_heketi_user_key }}" +openshift_storage_glusterfs_registry_heketi_admin_key: "{{ 32 | oo_generate_secret }}" +openshift_storage_glusterfs_registry_heketi_user_key: "{{ 32 | oo_generate_secret }}" openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}" openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}" openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index cb0fc0fbb..efcef8168 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -30,7 +30,7 @@ - kind: "sa" name: "heketi-{{ glusterfs_name }}-service-account" - kind: "secret" - name: "heketi-{{ glusterfs_name }}-secret" + name: "heketi-{{ glusterfs_name }}-user-secret" failed_when: False when: glusterfs_heketi_wipe @@ -168,3 +168,14 @@ when: - glusterfs_heketi_is_native - glusterfs_heketi_is_missing + +- name: Create heketi user secret + oc_secret: + namespace: "{{ glusterfs_namespace }}" + state: present + name: "heketi-{{ glusterfs_name }}-user-secret" + type: "kubernetes.io/glusterfs" + force: True + contents: + - path: key + data: "{{ glusterfs_heketi_user_key }}" -- cgit v1.2.3 From 46a75e7fac87d8017ecfc84896aa2df4d99d207a Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 31 May 2017 12:08:14 -0500 Subject: GlusterFS: Create a StorageClass if specified Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/README.md | 7 +++++++ roles/openshift_storage_glusterfs/defaults/main.yml | 2 ++ .../tasks/glusterfs_common.yml | 15 +++++++++++++++ .../tasks/glusterfs_config.yml | 1 + .../tasks/glusterfs_registry.yml | 1 + .../templates/v3.6/glusterfs-storageclass.yml.j2 | 10 ++++++++++ 6 files changed, 36 insertions(+) create mode 100644 roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md index 5783bcbd9..62fc35299 100644 --- a/roles/openshift_storage_glusterfs/README.md +++ b/roles/openshift_storage_glusterfs/README.md @@ -22,6 +22,11 @@ As part of the configuration, a particular GlusterFS cluster may be specified to provide backend storage for a natively-hosted Docker registry. +Unless configured otherwise, a StorageClass will be automatically +created for each non-registry GlusterFS cluster. This will allow +applications which can mount PersistentVolumes to request +dynamically-provisioned GlusterFS volumes. + Requirements ------------ @@ -75,6 +80,7 @@ GlusterFS cluster into a new or existing OpenShift cluster: | openshift_storage_glusterfs_is_native | True | GlusterFS should be containerized | openshift_storage_glusterfs_name | 'storage' | A name to identify the GlusterFS cluster, which will be used in resource names | openshift_storage_glusterfs_nodeselector | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name +| openshift_storage_glusterfs_storageclass | True | Automatically create a StorageClass for each GlusterFS cluster | openshift_storage_glusterfs_image | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7' | openshift_storage_glusterfs_version | 'latest' | Container image version to use for GlusterFS pods | openshift_storage_glusterfs_wipe | False | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.** @@ -98,6 +104,7 @@ are an exception: |-------------------------------------------------------|-----------------------|-----------------------------------------| | openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' | openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters +| openshift_storage_glusterfs_registry_storageclass | False | It is recommended to not create a StorageClass for GlusterFS clusters serving registry storage, so as to avoid performance penalties | openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated | Separate from the above | openshift_storage_glusterfs_registry_heketi_user_key | auto-generated | Separate from the above diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml index b6d06c086..468877e57 100644 --- a/roles/openshift_storage_glusterfs/defaults/main.yml +++ b/roles/openshift_storage_glusterfs/defaults/main.yml @@ -4,6 +4,7 @@ openshift_storage_glusterfs_namespace: 'default' openshift_storage_glusterfs_is_native: True openshift_storage_glusterfs_name: 'storage' openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host" +openshift_storage_glusterfs_storageclass: True openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}" openshift_storage_glusterfs_version: 'latest' openshift_storage_glusterfs_wipe: False @@ -23,6 +24,7 @@ openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.na openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}" openshift_storage_glusterfs_registry_name: 'registry' openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host" +openshift_storage_glusterfs_registry_storageclass: False openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}" openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}" openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index efcef8168..72d5629ec 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -179,3 +179,18 @@ contents: - path: key data: "{{ glusterfs_heketi_user_key }}" + +- name: Generate GlusterFS StorageClass file + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2" + dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + +- name: Create GlusterFS StorageClass + oc_obj: + state: present + kind: storageclass + name: "glusterfs-{{ glusterfs_name }}" + files: + - "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + when: + - glusterfs_storageclass diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 4c693528f..aa303d126 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -5,6 +5,7 @@ glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}" glusterfs_name: "{{ openshift_storage_glusterfs_name }}" glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}" + glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}" glusterfs_image: "{{ openshift_storage_glusterfs_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_version }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 7bfa0436c..59fdae94c 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -5,6 +5,7 @@ glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}" glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}" glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}" + glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}" glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}" glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}" glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}" diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 new file mode 100644 index 000000000..9b8fae310 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: glusterfs-{{ glusterfs_name }} +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://{{ glusterfs_heketi_url }}:8081" + secretNamespace: "{{ glusterfs_namespace }}" + secretName: "heketi-{{ glusterfs_name }}-user-secret" -- cgit v1.2.3 From 73820278c8c1ca1bc83e1c69f4939c0a48d88e46 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Wed, 31 May 2017 15:01:05 -0500 Subject: GlusterFS: Skip heketi-cli install on Atomic Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 72d5629ec..17b1eeb89 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -10,6 +10,11 @@ - name: Make sure heketi-client is installed package: name=heketi-client state=present + when: not openshift.common.is_atomic | bool + +- name: Verify heketi-cli is installed + shell: "command -v heketi-cli >/dev/null 2>&1 || { echo >&2 'ERROR: heketi-cli must be installed.'; exit 1; }" + changed_when: False - name: Delete pre-existing heketi resources oc_obj: -- cgit v1.2.3 From ddc472dbb4a54e15f07f6d947bda943a9a919da6 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Thu, 1 Jun 2017 10:47:20 -0500 Subject: GlusterFS: Fix bugs in wipe Signed-off-by: Jose A. Rivera --- roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml | 6 +++--- roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 17b1eeb89..8cb780507 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -5,9 +5,6 @@ name: "{{ glusterfs_namespace }}" when: glusterfs_is_native or glusterfs_heketi_is_native -- include: glusterfs_deploy.yml - when: glusterfs_is_native - - name: Make sure heketi-client is installed package: name=heketi-client state=present when: not openshift.common.is_atomic | bool @@ -63,6 +60,9 @@ retries: "{{ (glusterfs_timeout | int / 10) | int }}" when: glusterfs_heketi_wipe +- include: glusterfs_deploy.yml + when: glusterfs_is_native + - name: Create heketi service account oc_serviceaccount: namespace: "{{ glusterfs_namespace }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml index bb831147d..ea4dcc510 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -41,11 +41,16 @@ failed_when: False when: glusterfs_wipe - # Runs "vgremove -fy ; pvremove -fy " for every device found to be a physical volume. + # Runs "lvremove -ff ; vgremove -fy ; pvremove -fy " for every device found to be a physical volume. - name: Clear GlusterFS storage device contents - shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}" + shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}lvremove -ff {{ fields[1] }}; vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}" delegate_to: "{{ item.item }}" with_items: "{{ devices_info.results }}" + register: clear_devices + until: + - "'contains a filesystem in use' not in clear_devices.stderr" + delay: 1 + retries: 30 when: - glusterfs_wipe - item.stdout_lines | count > 0 -- cgit v1.2.3 From 97e1bc9436a757b2ffd802de531073f4b1b95c48 Mon Sep 17 00:00:00 2001 From: "Jose A. Rivera" Date: Fri, 9 Jun 2017 17:36:48 -0500 Subject: GlusterFS: Remove requirement for heketi-cli Signed-off-by: Jose A. Rivera --- .../files/v3.6/deploy-heketi-template.yml | 8 ++++ .../tasks/glusterfs_common.yml | 51 +++++++--------------- .../tasks/glusterfs_registry.yml | 4 +- .../tasks/heketi_deploy_part1.yml | 11 +++++ .../tasks/heketi_deploy_part2.yml | 27 ++++-------- 5 files changed, 46 insertions(+), 55 deletions(-) diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml index c67e05254..81b4fa5dc 100644 --- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml +++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml @@ -85,6 +85,8 @@ objects: volumeMounts: - name: db mountPath: /var/lib/heketi + - name: topology + mountPath: ${TOPOLOGY_PATH} readinessProbe: timeoutSeconds: 3 initialDelaySeconds: 3 @@ -99,6 +101,9 @@ objects: port: 8080 volumes: - name: db + - name: topology + secret: + secretName: heketi-${CLUSTER_NAME}-topology-secret parameters: - name: HEKETI_USER_KEY displayName: Heketi User Secret @@ -119,3 +124,6 @@ parameters: - name: CLUSTER_NAME displayName: GlusterFS cluster name value: glusterfs +- name: TOPOLOGY_PATH + displayName: heketi topology file location + required: True diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 8cb780507..829c1f51b 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -5,14 +5,6 @@ name: "{{ glusterfs_namespace }}" when: glusterfs_is_native or glusterfs_heketi_is_native -- name: Make sure heketi-client is installed - package: name=heketi-client state=present - when: not openshift.common.is_atomic | bool - -- name: Verify heketi-cli is installed - shell: "command -v heketi-cli >/dev/null 2>&1 || { echo >&2 'ERROR: heketi-cli must be installed.'; exit 1; }" - changed_when: False - - name: Delete pre-existing heketi resources oc_obj: namespace: "{{ glusterfs_namespace }}" @@ -25,6 +17,8 @@ selector: "deploy-heketi" - kind: "svc" name: "heketi-storage-endpoints" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-topology-secret" - kind: "template,route,service,dc" name: "heketi-{{ glusterfs_name }}" - kind: "svc" @@ -122,48 +116,35 @@ # heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0" +- name: Generate topology file + template: + src: "{{ openshift.common.examples_content_version }}/topology.json.j2" + dest: "{{ mktemp.stdout }}/topology.json" + when: + - glusterfs_heketi_topology_load + - include: heketi_deploy_part1.yml when: - glusterfs_heketi_is_native - glusterfs_heketi_deploy_is_missing - glusterfs_heketi_is_missing -- name: Determine heketi URL - oc_obj: - namespace: "{{ glusterfs_namespace }}" - state: list - kind: ep - selector: "glusterfs in (deploy-heketi-{{ glusterfs_name }}-service, heketi-{{ glusterfs_name }}-service)" - register: heketi_url - until: - - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" - - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''" - delay: 10 - retries: "{{ (glusterfs_timeout | int / 10) | int }}" - when: - - glusterfs_heketi_is_native - - glusterfs_heketi_url is undefined - - name: Set heketi URL set_fact: - glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}" + glusterfs_heketi_url: "localhost:8080" when: - glusterfs_heketi_is_native - - glusterfs_heketi_url is undefined + +- name: Set heketi-cli command + set_fact: + glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}'" - name: Verify heketi service - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list" + command: "{{ glusterfs_heketi_client }} cluster list" changed_when: False -- name: Generate topology file - template: - src: "{{ openshift.common.examples_content_version }}/topology.json.j2" - dest: "{{ mktemp.stdout }}/topology.json" - when: - - glusterfs_heketi_topology_load - - name: Load heketi topology - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' topology load --json={{ mktemp.stdout }}/topology.json 2>&1" + command: "{{ glusterfs_heketi_client }} topology load --json={{ mktemp.stdout }}/topology.json 2>&1" register: topology_load failed_when: "topology_load.rc != 0 or 'Unable' in topology_load.stdout" when: diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index 59fdae94c..4c6891eeb 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -66,9 +66,9 @@ - "{{ mktemp.stdout }}/glusterfs-registry-service.yml" - name: Check if GlusterFS registry volume exists - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume list" + command: "{{ glusterfs_heketi_client }} volume list" register: registry_volume - name: Create GlusterFS registry volume - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}" + command: "{{ glusterfs_heketi_client }} volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}" when: "openshift.hosted.registry.storage.glusterfs.path not in registry_volume.stdout" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml index fc4fbe7e1..318d34b5d 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -6,6 +6,16 @@ with_items: - "deploy-heketi-template.yml" +- name: Create heketi topology secret + oc_secret: + namespace: "{{ glusterfs_namespace }}" + state: present + name: "heketi-{{ glusterfs_name }}-topology-secret" + force: True + files: + - name: topology.json + path: "{{ mktemp.stdout }}/topology.json" + - name: Create deploy-heketi template oc_obj: namespace: "{{ glusterfs_namespace }}" @@ -27,6 +37,7 @@ HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" CLUSTER_NAME: "{{ glusterfs_name }}" + TOPOLOGY_PATH: "{{ mktemp.stdout }}" - name: Wait for deploy-heketi pod oc_obj: diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml index 80aa1f2db..3a9619d9d 100644 --- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -1,8 +1,10 @@ --- - name: Create heketi DB volume - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' setup-openshift-heketi-storage --listfile {{ mktemp.stdout }}/heketi-storage.json" + command: "{{ glusterfs_heketi_client }} setup-openshift-heketi-storage --listfile /tmp/heketi-storage.json" register: setup_storage - failed_when: False + +- name: Copy heketi-storage list + shell: "{{ openshift.common.client_binary }} rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json" # This is used in the subsequent task - name: Copy the admin client config @@ -48,6 +50,8 @@ selector: "deploy-heketi" - kind: "svc" name: "heketi-storage-endpoints" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-topology-secret" - name: Generate heketi endpoints template: @@ -118,23 +122,10 @@ delay: 10 retries: "{{ (glusterfs_timeout | int / 10) | int }}" -- name: Determine heketi URL - oc_obj: - namespace: "{{ glusterfs_namespace }}" - state: list - kind: ep - selector: "glusterfs=heketi-{{ glusterfs_name }}-service" - register: heketi_url - until: - - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''" - - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''" - delay: 10 - retries: "{{ (glusterfs_timeout | int / 10) | int }}" - -- name: Set heketi URL +- name: Set heketi-cli command set_fact: - glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}" + glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'" - name: Verify heketi service - command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list" + command: "{{ glusterfs_heketi_client }} cluster list" changed_when: False -- cgit v1.2.3