7 files changed, 32 insertions, 12 deletions
diff --git a/playbooks/aws/openshift-cluster/addNodes.yml b/playbooks/aws/openshift-cluster/add_nodes.yml
index 3d88e6b23..3d88e6b23 100644
--- a/playbooks/aws/openshift-cluster/addNodes.yml
+++ b/playbooks/aws/openshift-cluster/add_nodes.yml
diff --git a/playbooks/common/openshift-cluster/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
index b46407ed7..3a1a8ebb1 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/versions.sh
+++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
@@ -1,8 +1,8 @@
 #!/bin/bash
 
-yum_installed=$(yum list installed -e 0 -q "$@" 2>&1 | tail -n +2 | awk '{ print $2 }' | tr '\n' ' ')
+yum_installed=$(yum list installed -e 0 -q "$@" 2>&1 | tail -n +2 | awk '{ print $2 }' | sort -r | tr '\n' ' ')
 
-yum_available=$(yum list available -e 0 -q "$@" 2>&1 | tail -n +2 | grep -v 'el7ose' | awk '{ print $2 }' | tr '\n' ' ')
+yum_available=$(yum list available -e 0 -q "$@" 2>&1 | tail -n +2 | grep -v 'el7ose' | awk '{ print $2 }' | sort -r | tr '\n' ' ')
 
 echo "---"
 echo "curr_version: ${yum_installed}"
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index 2a4eecad9..8ec379109 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -422,6 +422,24 @@
   - name: Ensure node service enabled
     service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
 
+  - name: Install Ceph storage plugin dependencies
+    action: "{{ ansible_pkg_mgr }} name=ceph-common state=present"
+
+  - name: Install GlusterFS storage plugin dependencies
+    action: "{{ ansible_pkg_mgr }} name=glusterfs-fuse state=present"
+
+  - name: Set sebooleans to allow gluster storage plugin access from containers
+    seboolean:
+      name: "{{ item }}"
+      state: yes
+      persistent: yes
+    when: ansible_selinux and ansible_selinux.status == "enabled"
+    with_items:
+    - virt_use_fusefs
+    - virt_sandbox_use_fusefs
+    register: sebool_result
+    failed_when: "'state' not in sebool_result and 'msg' in sebool_result and 'SELinux boolean {{ item }} does not exist' not in sebool_result.msg"
+
   - set_fact:
       node_update_complete: True
 
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 9a5ae0e6b..d23a54511 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -89,7 +89,6 @@
   roles:
   - etcd
   - role: nickhammond.logrotate
-    when: not openshift.common.is_containerized | bool
 
 - name: Delete temporary directory on localhost
   hosts: localhost
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 9f8443599..a0d21451f 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -86,6 +86,7 @@
     etcd_generated_certs_dir: /etc/etcd/generated_certs
     etcd_needing_client_certs: "{{ hostvars
                                    | oo_select_keys(groups['oo_masters_to_config'])
+                                   | default([])
                                    | oo_filter_list(filter_attr='etcd_client_certs_missing') }}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
   roles:
@@ -337,9 +338,10 @@
   roles:
   - openshift_master
   - role: nickhammond.logrotate
-    when: not openshift.common.is_containerized | bool
   - role: fluentd_master
     when: openshift.common.use_fluentd | bool
+  - role: nuage_master
+    when: openshift.common.use_nuage | bool
   post_tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml
index 052892863..02449e40d 100644
--- a/playbooks/common/openshift-master/restart.yml
+++ b/playbooks/common/openshift-master/restart.yml
@@ -77,13 +77,6 @@
     when: openshift.master.cluster_method | default(None) == 'pacemaker'
     failed_when: false
     changed_when: false
-  # Any master which did not report 'active' or 'inactive' is likely
-  # unhealthy.  Other possible states are 'unknown' or 'failed'.
-  - fail:
-      msg: >
-        Got invalid service state from {{ openshift.common.service_type }}-master
-        on {{ inventory_hostname }}. Please verify pacemaker cluster.
-    when: openshift.master.cluster_method | default(None) == 'pacemaker' and active_check_output.stdout not in ['active', 'inactive']
   - set_fact:
       is_active: "{{ active_check_output.stdout == 'active' }}"
     when: openshift.master.cluster_method | default(None) == 'pacemaker'
@@ -92,6 +85,13 @@
   hosts: localhost
   become: no
   tasks:
+  - fail:
+      msg: >
+        Did not receive active status from any masters. Please verify pacemaker cluster.
+    when: "{{ hostvars[groups.oo_first_master.0].openshift.master.cluster_method | default(None) == 'pacemaker' and 'True' not in (hostvars
+              | oo_select_keys(groups['oo_masters_to_config'])
+              | oo_collect('is_active')
+              | list) }}"
   - name: Evaluate oo_active_masters
     add_host:
       name: "{{ item }}"
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 1d31657ed..dc3c25107 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -211,8 +211,9 @@
   roles:
   - role: flannel
     when: openshift.common.use_flannel | bool
+  - role: nuage_node
+    when: openshift.common.use_nuage | bool
   - role: nickhammond.logrotate
-    when: not openshift.common.is_containerized | bool
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool
   tasks: