Merge pull request #912 from twiest/master

sync master -> prod branch

8 files changed, 164 insertions, 17 deletions

diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml
index e0dbad900..565bb20a8 100644
--- a/playbooks/adhoc/uninstall.yml
+++ b/playbooks/adhoc/uninstall.yml
@@ -103,7 +103,7 @@
     - shell: find /var/lib/openshift/openshift.local.volumes -type d -exec umount {} \; 2>/dev/null || true
       changed_when: False
 
-    - shell: docker rm -f "{{ item }}"-master "{{ item }}"-node 
+    - shell: docker rm -f "{{ item }}"-master "{{ item }}"-node
       changed_when: False
       failed_when: False
       with_items:
@@ -152,10 +152,14 @@
         - /etc/sysconfig/atomic-enterprise-master
         - /etc/sysconfig/atomic-enterprise-node
         - /etc/sysconfig/atomic-openshift-master
+        - /etc/sysconfig/atomic-openshift-master-api
+        - /etc/sysconfig/atomic-openshift-master-controllers
         - /etc/sysconfig/atomic-openshift-node
         - /etc/sysconfig/openshift-master
         - /etc/sysconfig/openshift-node
         - /etc/sysconfig/origin-master
+        - /etc/sysconfig/origin-master-api
+        - /etc/sysconfig/origin-master-controllers
         - /etc/sysconfig/origin-node
         - /root/.kube
         - /run/openshift-sdn
@@ -165,6 +169,16 @@
         - /var/lib/openshift
         - /var/lib/origin
         - /var/lib/pacemaker
+        - /usr/lib/systemd/system/atomic-openshift-master-api.service
+        - /usr/lib/systemd/system/atomic-openshift-master-controllers.service
+        - /usr/lib/systemd/system/origin-master-api.service
+        - /usr/lib/systemd/system/origin-master-controllers.service
+
+    # Since we are potentially removing the systemd unit files for separated
+    # master-api and master-controllers services, so we need to reload the
+    # systemd configuration manager
+    - name: Reload systemd manager configuration
+      command: systemctl daemon-reload
 
     - name: restart docker
       service: name=docker state=restarted
diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
index ed4ab6d1b..b5459f312 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
+++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
@@ -83,7 +83,7 @@ def get(obj, *paths):
 
 
 # pylint: disable=too-many-arguments
-def pretty_print_errors(namespace, kind, item_name, container_name, port_name, valid):
+def pretty_print_errors(namespace, kind, item_name, container_name, invalid_label, port_name, valid):
     """
     Prints out results in human friendly way.
 
@@ -93,15 +93,16 @@ def pretty_print_errors(namespace, kind, item_name, container_name, port_name, v
        - `item_name`: Name of the resource
        - `container_name`: Name of the container. May be "" when kind=Service.
        - `port_name`: Name of the port
+       - `invalid_label`: The label of the invalid port. Port.name/targetPort
        - `valid`: True if the port is valid
     """
     if not valid:
         if len(container_name) > 0:
-            print('%s/%s -n %s (Container="%s" Port="%s")' % (
-                kind, item_name, namespace, container_name, port_name))
+            print('%s/%s -n %s (Container="%s" %s="%s")' % (
+                kind, item_name, namespace, container_name, invalid_label, port_name))
         else:
-            print('%s/%s -n %s (Port="%s")' % (
-                kind, item_name, namespace, port_name))
+            print('%s/%s -n %s (%s="%s")' % (
+                kind, item_name, namespace, invalid_label, port_name))
 
 
 def print_validation_header():
@@ -160,7 +161,7 @@ def main():
                         print_validation_header()
                     pretty_print_errors(
                         namespace, kind, item_name,
-                        container_name, port_name, valid)
+                        container_name, "Port.name", port_name, valid)
 
     # Services follow a different flow
     for item in list_items('services'):
@@ -176,7 +177,8 @@ def main():
                 first_error = False
                 print_validation_header()
             pretty_print_errors(
-                namespace, "services", item_name, "", port_name, valid)
+                namespace, "services", item_name, "",
+                "targetPort", port_name, valid)
 
     # If we had at least 1 error then exit with 1
     if not first_error:
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index 78797f8b8..eea147229 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -10,7 +10,7 @@
   roles:
   - openshift_facts
 
-- name: Evaluate etcd_hosts_to_backup
+- name: Evaluate additional groups for upgrade
   hosts: localhost
   tasks:
   - name: Evaluate etcd_hosts_to_backup
@@ -52,7 +52,7 @@
 
 
 - name: Verify upgrade can proceed
-  hosts: masters:nodes
+  hosts: oo_masters_to_config:oo_nodes_to_config
   tasks:
   - name: Clean yum cache
     command: yum clean all
@@ -78,6 +78,29 @@
       msg: Atomic OpenShift 3.1 packages not found
     when: g_aos_versions.curr_version | version_compare('3.0.2.900','<') and (g_aos_versions.avail_version is none or g_aos_versions.avail_version | version_compare('3.0.2.900','<'))
 
+  - set_fact:
+      pre_upgrade_complete: True
+
+
+##############################################################################
+# Gate on pre-upgrade checks
+##############################################################################
+- name: Gate on pre-upgrade checks
+  hosts: localhost
+  vars:
+    pre_upgrade_hosts: "{{ groups.oo_masters_to_config | union(groups.oo_nodes_to_config) }}"
+  tasks:
+  - set_fact:
+      pre_upgrade_completed: "{{ hostvars
+                                 | oo_select_keys(pre_upgrade_hosts)
+                                 | oo_collect('inventory_hostname', {'pre_upgrade_complete': true}) }}"
+  - set_fact:
+      pre_upgrade_failed: "{{ pre_upgrade_hosts | difference(pre_upgrade_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following hosts did not complete pre-upgrade checks: {{ pre_upgrade_failed | join(',') }}"
+    when: pre_upgrade_failed | length > 0
+
+
 
 ###############################################################################
 # Backup etcd
@@ -90,6 +113,7 @@
   roles:
   - openshift_facts
   tasks:
+  # Ensure we persist the etcd role for this host in openshift_facts
   - openshift_facts:
       role: etcd
       local_facts: {}
@@ -134,11 +158,32 @@
       etcdctl backup --data-dir={{ openshift.etcd.etcd_data_dir }}
       --backup-dir={{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}
 
+  - set_fact:
+      etcd_backup_complete: True
+
   - name: Display location of etcd backup
     debug:
       msg: "Etcd backup created in {{ openshift.common.data_dir }}/etcd-backup-{{ timestamp }}"
 
 
+##############################################################################
+# Gate on etcd backup
+##############################################################################
+- name: Gate on etcd backup
+  hosts: localhost
+  tasks:
+  - set_fact:
+      etcd_backup_completed: "{{ hostvars
+                                 | oo_select_keys(groups.etcd_hosts_to_backup)
+                                 | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}"
+  - set_fact:
+      etcd_backup_failed: "{{ groups.etcd_hosts_to_backup | difference(etcd_backup_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}"
+    when: etcd_backup_failed | length > 0
+
+
+
 ###############################################################################
 # Upgrade Masters
 ###############################################################################
@@ -152,7 +197,7 @@
     changed_when: False
 
 - name: Update deployment type
-  hosts: OSEv3
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
   roles:
   - openshift_facts
   post_tasks:
@@ -161,6 +206,16 @@
       local_facts:
         deployment_type: "{{ deployment_type }}"
 
+- name: Update master facts
+  hosts: oo_masters_to_config
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        cluster_method: "{{ openshift_master_cluster_method | default(None) }}"
+
 - name: Upgrade master packages and configuration
   hosts: oo_masters_to_config
   vars:
@@ -290,6 +345,30 @@
     changed_when: False
 
 
+- name: Set master update status to complete
+  hosts: oo_masters_to_config
+  tasks:
+  - set_fact:
+      master_update_complete: True
+
+
+##############################################################################
+# Gate on master update complete
+##############################################################################
+- name: Gate on master update
+  hosts: localhost
+  tasks:
+  - set_fact:
+      master_update_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_masters_to_config)
+                                 | oo_collect('inventory_hostname', {'master_update_complete': true}) }}"
+  - set_fact:
+      master_update_failed: "{{ groups.oo_masters_to_config | difference(master_update_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following masters did not finish updating: {{ master_update_failed | join(',') }}"
+    when: master_update_failed | length > 0
+
+
 ###############################################################################
 # Upgrade Nodes
 ###############################################################################
@@ -309,6 +388,26 @@
   - name: Ensure node service enabled
     service: name="{{ openshift.common.service_type }}-node" state=started enabled=yes
 
+  - set_fact:
+      node_update_complete: True
+
+
+##############################################################################
+# Gate on nodes update
+##############################################################################
+- name: Gate on nodes update
+  hosts: localhost
+  tasks:
+  - set_fact:
+      node_update_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_nodes_to_config)
+                                 | oo_collect('inventory_hostname', {'node_update_complete': true}) }}"
+  - set_fact:
+      node_update_failed: "{{ groups.oo_nodes_to_config | difference(node_update_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following nodes did not finish updating: {{ node_update_failed | join(',') }}"
+    when: node_update_failed | length > 0
+
 
 ###############################################################################
 # Post upgrade - Reconcile Cluster Roles and Cluster Role Bindings
@@ -356,6 +455,28 @@
     when: openshift_master_ha | bool
     run_once: true
 
+  - set_fact:
+      reconcile_complete: True
+
+
+##############################################################################
+# Gate on reconcile
+##############################################################################
+- name: Gate on reconcile
+  hosts: localhost
+  tasks:
+  - set_fact:
+      reconcile_completed: "{{ hostvars
+                                 | oo_select_keys(groups.oo_masters_to_config)
+                                 | oo_collect('inventory_hostname', {'reconcile_complete': true}) }}"
+  - set_fact:
+      reconcile_failed: "{{ groups.oo_masters_to_config | difference(reconcile_completed) }}"
+  - fail:
+      msg: "Upgrade cannot continue. The following masters did not finish reconciling: {{ reconcile_failed | join(',') }}"
+    when: reconcile_failed | length > 0
+
+
+
 
 ###############################################################################
 # Post upgrade - Upgrade default router, default registry and examples
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 1b3fba3aa..b1da85d5d 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -51,9 +51,6 @@
           console_url: "{{ openshift_master_console_url | default(None) }}"
           console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
           public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
-      - role: etcd
-        local_facts: {}
-        when: openshift.master.embedded_etcd | bool
   - name: Check status of external etcd certificatees
     stat:
       path: "{{ openshift.common.config_base }}/master/{{ item }}"
diff --git a/playbooks/libvirt/openshift-cluster/list.yml b/playbooks/libvirt/openshift-cluster/list.yml
index eaedc4d0d..5954bb01e 100644
--- a/playbooks/libvirt/openshift-cluster/list.yml
+++ b/playbooks/libvirt/openshift-cluster/list.yml
@@ -18,6 +18,12 @@
 
 - name: List Hosts
   hosts: oo_list_hosts
+
+- name: List Hosts
+  hosts: localhost
+  gather_facts: no
+  vars_files:
+  - vars.yml
   tasks:
   - debug:
-      msg: 'public:{{ansible_default_ipv4.address}} private:{{ansible_default_ipv4.address}}'
+      msg: "{{ hostvars | oo_select_keys(groups[scratch_group] | default([])) | oo_pretty_print_cluster }}"
diff --git a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
index 4b91c6da8..4825207c9 100644
--- a/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/libvirt/openshift-cluster/tasks/launch_instances.yml
@@ -81,7 +81,7 @@
     ansible_ssh_host: '{{ item.1 }}'
     ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
     ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    groups: 'tag_env-{{ cluster }}, tag_host-type-{{ type }}, tag_env-host-type-{{ cluster }}-openshift-{{ type }}'
+    groups: 'tag_env-{{ cluster }}, tag_host-type-{{ type }}, tag_env-host-type-{{ cluster }}-openshift-{{ type }}, tag_sub-host-type-{{ g_sub_host_type }}'
   with_together:
     - instances
     - ips
diff --git a/playbooks/libvirt/openshift-cluster/templates/domain.xml b/playbooks/libvirt/openshift-cluster/templates/domain.xml
index df200e374..870bcf2a6 100644
--- a/playbooks/libvirt/openshift-cluster/templates/domain.xml
+++ b/playbooks/libvirt/openshift-cluster/templates/domain.xml
@@ -6,6 +6,7 @@
       <ansible:tag>env-{{ cluster }}</ansible:tag>
       <ansible:tag>env-host-type-{{ cluster }}-openshift-{{ type }}</ansible:tag>
       <ansible:tag>host-type-{{ type }}</ansible:tag>
+      <ansible:tag>sub-host-type-{{ g_sub_host_type }}</ansible:tag>
     </ansible:tags>
   </metadata>
   <currentMemory unit='GiB'>1</currentMemory>
diff --git a/playbooks/openstack/openshift-cluster/list.yml b/playbooks/openstack/openshift-cluster/list.yml
index a75e350c7..fa194b072 100644
--- a/playbooks/openstack/openshift-cluster/list.yml
+++ b/playbooks/openstack/openshift-cluster/list.yml
@@ -19,6 +19,12 @@
 
 - name: List Hosts
   hosts: oo_list_hosts
+
+- name: List Hosts
+  hosts: localhost
+  gather_facts: no
+  vars_files:
+  - vars.yml
   tasks:
   - debug:
-      msg: 'public:{{ansible_ssh_host}} private:{{ansible_default_ipv4.address}}'
+      msg: "{{ hostvars | oo_select_keys(groups[scratch_group] | default([])) | oo_pretty_print_cluster }}"