Merge pull request #912 from twiest/master

sync master -> prod branch

2 files changed, 133 insertions, 10 deletions

diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
index ed4ab6d1b..b5459f312 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
+++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
@@ -83,7 +83,7 @@ def get(obj, *paths):
 
 
 # pylint: disable=too-many-arguments
-def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid):
+def pretty_print_errors(namespace, kind, item_name, container_name, invalid_label, port_name, valid):
     """
     Prints out results in human friendly way.
 
@@ -93,15 +93,16 @@ def pretty_print_errors(namespace, kind, item_name, container_name, port_name, v
        - `item_name`: Name of the resource
        - `container_name`: Name of the container. May be "" when kind=Service.
        - `port_name`: Name of the port
+       - `invalid_label`: The label of the invalid port. Port.name/targetPort
        - `valid`: True if the port is valid
     """
     if not valid:
         if len(container_name) > 0:
-            print('%s/%s -n %s (Container="%s" Port="%s")' % (
-                kind, item_name, namespace, container_name, port_name))
+            print('%s/%s -n %s (Container="%s" %s="%s")' % (
+                kind, item_name, namespace, container_name, invalid_label, port_name))
         else:
-            print('%s/%s -n %s (Port="%s")' % (
-                kind, item_name, namespace, port_name))
+            print('%s/%s -n %s (%s="%s")' % (
+                kind, item_name, namespace, invalid_label, port_name))
 
 
 def print_validation_header():
@@ -160,7 +161,7 @@ def main():
                         print_validation_header()
                     pretty_print_errors(
                         namespace, kind, item_name,
-                        container_name, port_name, valid)
+                        container_name, "Port.name", port_name, valid)
 
     # Services follow a different flow
     for item in list_items('services'):
@@ -176,7 +177,8 @@ def main():
                 first_error = False
                 print_validation_header()
             pretty_print_errors(
-                namespace, "services", item_name, "", port_name, valid)
+                namespace, "services", item_name, "",
+                "targetPort", port_name, valid)
 
     # If we had at least 1 error then exit with 1
     if not first_error:
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index 78797f8b8..eea147229 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -10,7 +10,7 @@
   roles:
   - openshift_facts
 
-- name: Evaluate etcd_hosts_to_backup
+- name: Evaluate additional groups for upgrade
   hosts: localhost
   tasks:
   - name: Evaluate etcd_hosts_to_backup
@@ -52,7 +52,7 @@
 
 
 - name: Verify upgrade can proceed
-  hosts: masters:nodes
+  hosts: oo_masters_to_config:oo_nodes_to_config
   tasks:
   - name: Clean yum cache
     command: yum clean all
@@ -78,6 +78,29 @@
       msg: Atomic OpenShift 3.1 packages not found
     when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
 
+  - set_fact:
+      pre_upgrade_complete: True
+
+
+##############################################################################
+# Gate on pre-upgrade checks
+##############################################################################
+- name: Gate on pre-upgrade checks
+  hosts: localhost
+  vars:
+    pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}"
+  tasks:
+  - set_fact:
+      pre_upgrade_completed: "{{ hostvars
+                                 | oo_select_keys(pre_upgrade_hosts)
+                                 | oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}"
+  - set_fact:
+      pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}"
+    when: pre_upgrade_failed | length > 0
+
+
 
 ###############################################################################
 # Backup etcd
@@ -90,6 +113,7 @@
   roles:
   - openshift_facts
   tasks:
+  # Ensure we persist the etcd role for this host in openshift_facts
   - openshift_facts:
       role: etcd
       local_facts: {}
@@ -134,11 +158,32 @@
       etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }}
       --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
 
+  - set_fact:
+      etcd_backup_complete: True
+
   - name: Display location of etcd backup
     debug:
       msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
 
 
+##############################################################################
+# Gate on etcd backup
+##############################################################################
+- name: Gate on etcd backup
+  hosts: localhost
+  tasks:
+  - set_fact:
+      etcd_backup_completed: "{{ hostvars
+                                 | oo_select_keys(groups.etcd_hosts_to_backup)
+                                 | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}"
+  - set_fact:
+      etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
+    when: etcd_backup_failed | length > 0
+
+
+
 ###############################################################################
 # Upgrade Masters
 ###############################################################################
@@ -152,7 +197,7 @@
     changed_when: False
 
 - name: Update deployment type
-  hosts: OSEv3
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
   roles:
   - openshift_facts
   post_tasks:
@@ -161,6 +206,16 @@
       local_facts:
         deployment_type: "{{ deployment_type }}"
 
+- name: Update master facts
+  hosts: oo_masters_to_config
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        cluster_method: "{{ openshift_master_cluster_method | default(None) }}"
+
 - name: Upgrade master packages and configuration
   hosts: oo_masters_to_config
   vars:
@@ -290,6 +345,30 @@
     changed_when: False
 
 
+- name: Set master update status to complete
+  hosts: oo_masters_to_config
+  tasks:
+  - set_fact:
+      master_update_complete: True
+
+
+##############################################################################
+# Gate on master update complete
+##############################################################################
+- name: Gate on master update
+  hosts: localhost
+  tasks:
+  - set_fact:
+      master_update_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_masters_to_config)
+                                 | oo_collect('inventory_hostname', {'master_update_complete': true}) }}"
+  - set_fact:
+      master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}"
+    when: master_update_failed | length > 0
+
+
 ###############################################################################
 # Upgrade Nodes
 ###############################################################################
@@ -309,6 +388,26 @@
   - name: Ensure node service enabled
     service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
 
+  - set_fact:
+      node_update_complete: True
+
+
+##############################################################################
+# Gate on nodes update
+##############################################################################
+- name: Gate on nodes update
+  hosts: localhost
+  tasks:
+  - set_fact:
+      node_update_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_nodes_to_config)
+                                 | oo_collect('inventory_hostname', {'node_update_complete': true}) }}"
+  - set_fact:
+      node_update_failed: "{{ groups.oo_nodes_to_config | difference(node_update_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following nodes did not finish updating: {{ node_update_failed | join(',') }}"
+    when: node_update_failed | length > 0
+
 
 ###############################################################################
 # Post upgrade - Reconcile Cluster Roles and Cluster Role Bindings
@@ -356,6 +455,28 @@
     when: openshift_master_ha | bool
     run_once: true
 
+  - set_fact:
+      reconcile_complete: True
+
+
+##############################################################################
+# Gate on reconcile
+##############################################################################
+- name: Gate on reconcile
+  hosts: localhost
+  tasks:
+  - set_fact:
+      reconcile_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_masters_to_config)
+                                 | oo_collect('inventory_hostname', {'reconcile_complete': true}) }}"
+  - set_fact:
+      reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}"
+    when: reconcile_failed | length > 0
+
+
+
 
 ###############################################################################
 # Post upgrade - Upgrade default router, default registry and examples