Merge pull request #1048 from twiest/prod

Sync master -> Prod

26 files changed, 402 insertions, 91 deletions

diff --git a/playbooks/adhoc/bootstrap-fedora.yml b/playbooks/adhoc/bootstrap-fedora.yml
new file mode 100644
index 000000000..de9f36c8a
--- /dev/null
+++ b/playbooks/adhoc/bootstrap-fedora.yml
@@ -0,0 +1,5 @@
+- hosts: OSv3
+  gather_facts: false
+  tasks:
+    - name: install python and deps for ansible modules
+      raw: dnf install -y python2 python2-dnf libselinux-python libsemanage-python
diff --git a/playbooks/adhoc/uninstall.yml b/playbooks/adhoc/uninstall.yml
index 565bb20a8..08a2ea6fb 100644
--- a/playbooks/adhoc/uninstall.yml
+++ b/playbooks/adhoc/uninstall.yml
@@ -48,7 +48,39 @@
         - pcsd
 
     - yum: name={{ item }} state=absent
-      when: not is_atomic | bool
+      when: ansible_pkg_mgr == "yum" and not is_atomic | bool
+      with_items:
+        - atomic-enterprise
+        - atomic-enterprise-master
+        - atomic-enterprise-node
+        - atomic-enterprise-sdn-ovs
+        - atomic-openshift
+        - atomic-openshift-clients
+        - atomic-openshift-master
+        - atomic-openshift-node
+        - atomic-openshift-sdn-ovs
+        - corosync
+        - etcd
+        - openshift
+        - openshift-master
+        - openshift-node
+        - openshift-sdn
+        - openshift-sdn-ovs
+        - openvswitch
+        - origin
+        - origin-clients
+        - origin-master
+        - origin-node
+        - origin-sdn-ovs
+        - pacemaker
+        - pcs
+        - tuned-profiles-atomic-enterprise-node
+        - tuned-profiles-atomic-openshift-node
+        - tuned-profiles-openshift-node
+        - tuned-profiles-origin-node
+
+    - dnf: name={{ item }} state=absent
+      when: ansible_pkg_mgr == "dnf" and not is_atomic | bool
       with_items:
         - atomic-enterprise
         - atomic-enterprise-master
@@ -111,12 +143,12 @@
         - atomic-enterprise
         - origin
 
-    - shell: docker ps -a | grep Exited | grep "{{ item }}" | awk '{print $1}'
+    - shell: docker ps -a | grep Exited | egrep "{{ item }}" | awk '{print $1}'
       changed_when: False
       failed_when: False
       register: exited_containers_to_delete
       with_items:
-        - aep3/aep
+        - aep3.*/aep
         - openshift3/ose
         - openshift/origin
 
@@ -125,13 +157,13 @@
       failed_when: False
       with_items: "{{ exited_containers_to_delete.results }}"
 
-    - shell: docker images | grep {{ item }} | awk '{ print $3 }'
+    - shell: docker images | egrep {{ item }} | awk '{ print $3 }'
       changed_when: False
       failed_when: False
       register: images_to_delete
       with_items:
-        - registry.access.redhat.com/openshift3
-        - registry.access.redhat.com/aep3
+        - registry\.access\..*redhat\.com/openshift3
+        - registry\.access\..*redhat\.com/aep3
         - docker.io/openshift
 
     - shell:  "docker rmi -f {{ item.stdout_lines | join(' ') }}"
@@ -161,6 +193,7 @@
         - /etc/sysconfig/origin-master-api
         - /etc/sysconfig/origin-master-controllers
         - /etc/sysconfig/origin-node
+        - /etc/systemd/system/atomic-openshift-node.service.wants
         - /root/.kube
         - /run/openshift-sdn
         - /usr/share/openshift/examples
@@ -180,5 +213,8 @@
     - name: Reload systemd manager configuration
       command: systemctl daemon-reload
 
+- hosts: nodes
+  sudo: yes
+  tasks:
     - name: restart docker
       service: name=docker state=restarted
diff --git a/playbooks/aws/openshift-cluster/addNodes.yml b/playbooks/aws/openshift-cluster/addNodes.yml
new file mode 100644
index 000000000..fff3e401b
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/addNodes.yml
@@ -0,0 +1,39 @@
+---
+- name: Launch instance(s)
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars_files:
+  - vars.yml
+  - ["vars.{{ deployment_type }}.{{ cluster_id }}.yml", vars.defaults.yml]
+  vars:
+    oo_extend_env: True
+  tasks:
+  - fail:
+      msg: Deployment type not supported for aws provider yet
+    when: deployment_type == 'enterprise'
+
+  - include: ../../common/openshift-cluster/tasks/set_node_launch_facts.yml
+    vars:
+      type: "compute"
+      count: "{{ num_nodes }}"
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ node_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
+
+  - include: ../../common/openshift-cluster/tasks/set_node_launch_facts.yml
+    vars:
+      type: "infra"
+      count: "{{ num_infra }}"
+  - include: tasks/launch_instances.yml
+    vars:
+      instances: "{{ node_names }}"
+      cluster: "{{ cluster_id }}"
+      type: "{{ k8s_type }}"
+      g_sub_host_type: "{{ sub_host_type }}"
+
+- include: scaleup.yml
+- include: list.yml
diff --git a/playbooks/aws/openshift-cluster/scaleup.yml b/playbooks/aws/openshift-cluster/scaleup.yml
new file mode 100644
index 000000000..4415700a3
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/scaleup.yml
@@ -0,0 +1,34 @@
+---
+
+- hosts: localhost
+  gather_facts: no
+  vars_files:
+  - vars.yml
+  tasks:
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
+  - name: Evaluate oo_hosts_to_update
+    add_host:
+      name: "{{ item }}"
+      groups: oo_hosts_to_update
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+    with_items: "{{ groups.nodes_to_add }}"
+
+- include: ../../common/openshift-cluster/update_repos_and_packages.yml
+
+- include: ../../common/openshift-cluster/scaleup.yml
+  vars:
+    g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-lb' }}"
+    g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
+    g_new_nodes_group: 'nodes_to_add'
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
+    g_nodeonmaster: true
+    openshift_cluster_id: "{{ cluster_id }}"
+    openshift_debug_level: 2
+    openshift_deployment_type: "{{ deployment_type }}"
+    openshift_hostname: "{{ ec2_private_ip_address }}"
+    openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
index 9c699120b..99f0577fc 100644
--- a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
@@ -20,10 +20,6 @@
                    | default(deployment_vars[deployment_type].image, true) }}"
   when: ec2_image is not defined and not ec2_image_name
 - set_fact:
-    ec2_instance_type: "{{ lookup('env', 'ec2_instance_type')
-                    | default(deployment_vars[deployment_type].type, true) }}"
-  when: ec2_instance_type is not defined
-- set_fact:
     ec2_keypair: "{{ lookup('env', 'ec2_keypair')
                     | default(deployment_vars[deployment_type].keypair, true) }}"
   when: ec2_keypair is not defined
@@ -37,25 +33,25 @@
   when: ec2_assign_public_ip is not defined
 
 - set_fact:
-    ec2_instance_type: "{{ ec2_master_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_instance_type: "{{ ec2_master_instance_type | default(lookup('env', 'ec2_master_instance_type') | default(lookup('env', 'ec2_instance_type') | default(deployment_vars[deployment_type].type, true), true), true) }}"
     ec2_security_groups: "{{ ec2_master_security_groups
                     | default(deployment_vars[deployment_type].security_groups, true) }}"
   when: host_type == "master" and sub_host_type == "default"
 
 - set_fact:
-    ec2_instance_type: "{{ ec2_etcd_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_instance_type: "{{ ec2_etcd_instance_type | default(lookup('env', 'ec2_etcd_instance_type') | default(lookup('env', 'ec2_instance_type') | default(deployment_vars[deployment_type].type, true), true), true) }}"
     ec2_security_groups: "{{ ec2_etcd_security_groups
                     | default(deployment_vars[deployment_type].security_groups, true)}}"
   when: host_type == "etcd" and sub_host_type == "default"
 
 - set_fact:
-    ec2_instance_type: "{{ ec2_infra_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_instance_type: "{{ ec2_infra_instance_type | default(lookup('env', 'ec2_infra_instance_type') | default(lookup('env', 'ec2_instance_type') | default(deployment_vars[deployment_type].type, true), true), true) }}"
     ec2_security_groups: "{{ ec2_infra_security_groups
                     | default(deployment_vars[deployment_type].security_groups, true) }}"
   when: host_type == "node" and sub_host_type == "infra"
 
 - set_fact:
-    ec2_instance_type: "{{ ec2_node_instance_type | default(deployment_vars[deployment_type].type, true) }}"
+    ec2_instance_type: "{{ ec2_node_instance_type | default(lookup('env', 'ec2_node_instance_type') | default(lookup('env', 'ec2_instance_type') | default(deployment_vars[deployment_type].type, true), true), true) }}"
     ec2_security_groups: "{{ ec2_node_security_groups
                     | default(deployment_vars[deployment_type].security_groups, true) }}"
   when: host_type == "node" and sub_host_type == "compute"
@@ -81,7 +77,6 @@
 
 - set_fact:
     latest_ami: "{{ ami_result.results | oo_ami_selector(ec2_image_name) }}"
-    user_data: "{{ lookup('template', '../templates/user_data.j2') }}"
     volume_defs:
       etcd:
         root:
@@ -97,6 +92,10 @@
           volume_size: "{{ lookup('env', 'os_master_root_vol_size') | default(25, true) }}"
           device_type: "{{ lookup('env', 'os_master_root_vol_type') | default('gp2', true) }}"
           iops: "{{ lookup('env', 'os_master_root_vol_iops') | default(500, true) }}"
+        docker:
+          volume_size: "{{ lookup('env', 'os_docker_vol_size') | default(10, true) }}"
+          device_type: "{{ lookup('env', 'os_docker_vol_type') | default('gp2', true) }}"
+          iops: "{{ lookup('env', 'os_docker_vol_iops') | default(500, true) }}"
       node:
         root:
           volume_size: "{{ lookup('env', 'os_node_root_vol_size') | default(85, true) }}"
@@ -121,7 +120,7 @@
     count: "{{ instances | length }}"
     vpc_subnet_id: "{{ ec2_vpc_subnet | default(omit, true) }}"
     assign_public_ip: "{{ ec2_assign_public_ip | default(omit, true) }}"
-    user_data: "{{ user_data }}"
+    user_data: "{{ lookup('template', '../templates/user_data.j2') }}"
     wait: yes
     instance_tags:
       created-by: "{{ created_by }}"
@@ -191,6 +190,22 @@
   - instances
   - ec2.instances
 
+- name: Add new instances to nodes_to_add group if needed
+  add_host:
+    hostname: "{{ item.0 }}"
+    ansible_ssh_host: "{{ item.1.dns_name }}"
+    ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+    ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+    groups: nodes_to_add
+    ec2_private_ip_address: "{{ item.1.private_ip }}"
+    ec2_ip_address: "{{ item.1.public_ip }}"
+    openshift_node_labels: "{{ node_label }}"
+    logrotate_scripts: "{{ logrotate }}"
+  with_together:
+  - instances
+  - ec2.instances
+  when: oo_extend_env is defined and oo_extend_env | bool
+
 - name: Wait for ssh
   wait_for: "port=22 host={{ item.dns_name }}"
   with_items: ec2.instances
diff --git a/playbooks/aws/openshift-cluster/templates/user_data.j2 b/playbooks/aws/openshift-cluster/templates/user_data.j2
index 82c2f4d57..3621a7d7d 100644
--- a/playbooks/aws/openshift-cluster/templates/user_data.j2
+++ b/playbooks/aws/openshift-cluster/templates/user_data.j2
@@ -1,5 +1,5 @@
 #cloud-config
-{% if type =='etcd' %}
+{% if type == 'etcd' and 'etcd' in volume_defs[type] %}
 cloud_config_modules:
 - disk_setup
 - mounts
@@ -19,7 +19,7 @@ fs_setup:
   partition: auto
 {% endif %}
 
-{% if type == 'node' %}
+{% if type in ['node', 'master'] and 'docker' in volume_defs[type] %}
 mounts:
 - [ xvdb ]
 - [ ephemeral0 ]
@@ -43,3 +43,10 @@ growpart:
 runcmd:
 - xfs_growfs /var
 {% endif %}
+
+{% if deployment_vars[deployment_type].sudo %}
+- path: /etc/sudoers.d/99-{{ deployment_vars[deployment_type].ssh_user }}-cloud-init-requiretty
+  permissions: 440
+  content: |
+    Defaults:{{ deployment_vars[deployment_type].ssh_user }} !requiretty
+{% endif %}
diff --git a/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
new file mode 100644
index 000000000..8cad51b5e
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -0,0 +1,33 @@
+---
+# This playbook upgrades an existing AWS cluster, leaving nodes untouched if used with an 'online' deployment type.
+# Usage:
+#  ansible-playbook playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml -e deployment_type=online -e cluster_id=<cluster_id>
+- hosts: localhost
+  gather_facts: no
+  vars_files:
+  - ../../vars.yml
+  - "../../vars.{{ deployment_type }}.{{ cluster_id }}.yml"
+
+  tasks:
+  - set_fact:
+      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
+      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
+
+  - set_fact:
+      tmp_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
+    when: deployment_type != 'online'
+
+- include: ../../../../common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+  vars:
+    g_etcd_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-etcd' }}"
+    g_lb_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-lb' }}"
+    g_masters_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-master' }}"
+    g_nodes_group: "{{ tmp_nodes_group | default('') }}"
+    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
+    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
+    g_nodeonmaster: true
+    openshift_cluster_id: "{{ cluster_id }}"
+    openshift_debug_level: 2
+    openshift_deployment_type: "{{ deployment_type }}"
+    openshift_hostname: "{{ ec2_private_ip_address }}"
+    openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/byo/openshift-cluster/scaleup.yml b/playbooks/byo/openshift-cluster/scaleup.yml
new file mode 100644
index 000000000..70644d427
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/scaleup.yml
@@ -0,0 +1,10 @@
+---
+- include: ../../common/openshift-cluster/scaleup.yml
+  vars:
+    g_etcd_group: "{{ 'etcd' }}"
+    g_masters_group: "{{ 'masters' }}"
+    g_new_nodes_group: "{{ 'new_nodes' }}"
+    g_lb_group: "{{ 'lb' }}"
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_debug_level: 2
+    openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/byo/openshift_facts.yml b/playbooks/byo/openshift_facts.yml
index 6d7c12fd4..babdfb952 100644
--- a/playbooks/byo/openshift_facts.yml
+++ b/playbooks/byo/openshift_facts.yml
@@ -1,7 +1,6 @@
 ---
 - name: Gather Cluster facts
-  hosts: all
-  gather_facts: no
+  hosts: OSEv3
   roles:
   - openshift_facts
   tasks:
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index a8bd634d3..482fa8441 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -6,6 +6,3 @@
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-node/config.yml
-  vars:
-    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
-    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].cluster_dns_ip }}"
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index 2bb69614f..34da372a4 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -12,8 +12,8 @@
     when: g_masters_group is not defined
 
   - fail:
-      msg: This playbook requires g_nodes_group to be set
-    when: g_nodes_group is not defined
+      msg: This playbook requires g_nodes_group or g_new_nodes_group to be set
+    when: g_nodes_group is not defined and g_new_nodes_group is not defined
 
   - fail:
       msg: This playbook requires g_lb_group to be set
@@ -35,14 +35,19 @@
       ansible_sudo: "{{ g_sudo | default(omit) }}"
     with_items: groups[g_masters_group] | default([])
 
+  # Use g_new_nodes_group if it exists otherwise g_nodes_group
+  - set_fact:
+      g_nodes_to_config: "{{ g_new_nodes_group | default(g_nodes_group | default([])) }}"
+
   - name: Evaluate oo_nodes_to_config
     add_host:
       name: "{{ item }}"
       groups: oo_nodes_to_config
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
-    with_items: groups[g_nodes_group] | default([])
+    with_items: groups[g_nodes_to_config] | default([])
 
+  # Skip adding the master to oo_nodes_to_config when g_new_nodes_group is
   - name: Evaluate oo_nodes_to_config
     add_host:
       name: "{{ item }}"
@@ -50,7 +55,7 @@
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_sudo: "{{ g_sudo | default(omit) }}"
     with_items: groups[g_masters_group] | default([])
-    when: g_nodeonmaster is defined and g_nodeonmaster == true
+    when: g_nodeonmaster | default(false) == true and g_new_nodes_group is not defined
 
   - name: Evaluate oo_first_etcd
     add_host:
diff --git a/playbooks/common/openshift-cluster/scaleup.yml b/playbooks/common/openshift-cluster/scaleup.yml
index 6d2777732..d2ba3fc7a 100644
--- a/playbooks/common/openshift-cluster/scaleup.yml
+++ b/playbooks/common/openshift-cluster/scaleup.yml
@@ -1,16 +1,6 @@
 ---
 - include: evaluate_groups.yml
-  vars:
-    g_etcd_group: "{{ 'etcd' }}"
-    g_masters_group: "{{ 'masters' }}"
-    g_nodes_group: "{{ 'nodes' }}"
-    g_lb_group: "{{ 'lb' }}"
-    openshift_cluster_id: "{{ cluster_id | default('default') }}"
-    openshift_debug_level: 2
-    openshift_deployment_type: "{{ deployment_type }}"
 
 - include: ../openshift-node/config.yml
   vars:
-    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
-    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
     openshift_deployment_type: "{{ deployment_type }}"
diff --git a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
index b5459f312..e5c958ebb 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
+++ b/playbooks/common/openshift-cluster/upgrades/files/pre-upgrade-check
@@ -111,13 +111,16 @@ def print_validation_header():
     overwhelming the user.
     """
     print """\
-At least one port name does not validate. Valid port names:
+At least one port name is invalid and must be corrected before upgrading.
+Please update or remove any resources with invalid port names.
 
-    * must be less that 16 chars
+  Valid port names must:
+
+    * be less that 16 characters
     * have at least one letter
-    * only a-z0-9-
-    * do not start or end with -
-    * Dashes may not be next to eachother ('--')
+    * contain only a-z0-9-
+    * not start or end with -
+    * not contain dashes next to each other ('--')
 """
 
 
@@ -142,9 +145,9 @@ def main():
     # Where the magic happens
     first_error = True
     for kind, path in [
+            ('deploymentconfigs', ("spec", "template", "spec", "containers")),
             ('replicationcontrollers', ("spec", "template", "spec", "containers")),
-            ('pods', ("spec", "containers")),
-            ('deploymentconfigs', ("spec", "template", "spec", "containers"))]:
+            ('pods', ("spec", "containers"))]:
         for item in list_items(kind):
             namespace = item["metadata"]["namespace"]
             item_name = item["metadata"]["name"]
diff --git a/playbooks/common/openshift-cluster/upgrades/files/versions.sh b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
index f90719cab..c7c966b60 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/versions.sh
+++ b/playbooks/common/openshift-cluster/upgrades/files/versions.sh
@@ -2,9 +2,9 @@
 
 yum_installed=$(yum list installed "$@" 2>&1 | tail -n +2 | grep -v 'Installed Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
 
-yum_available=$(yum list available "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
+yum_available=$(yum list available -q "$@" 2>&1 | tail -n +2 | grep -v 'Available Packages' | grep -v 'Red Hat Subscription Management' | grep -v 'el7ose' | grep -v 'Error:' | awk '{ print $2 }' | tr '\n' ' ')
 
 
 echo "---"
-echo "curr_version: ${yum_installed}" 
+echo "curr_version: ${yum_installed}"
 echo "avail_version: ${yum_available}"
diff --git a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
index a6721bb92..9a065fd1c 100755
--- a/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
+++ b/playbooks/common/openshift-cluster/upgrades/library/openshift_upgrade_config.py
@@ -78,6 +78,10 @@ def upgrade_master_3_0_to_3_1(ansible_module, config_base, backup):
         config['kubernetesMasterConfig'].pop('apiLevels')
         changes.append('master-config.yaml: removed kubernetesMasterConfig.apiLevels')
 
+    # Add masterCA to serviceAccountConfig
+    if 'serviceAccountConfig' in config and 'masterCA' not in config['serviceAccountConfig']:
+        config['serviceAccountConfig']['masterCA'] = config['oauthConfig'].get('masterCA', 'ca.crt')
+
     # Add proxyClientInfo to master-config
     if 'proxyClientInfo' not in config['kubernetesMasterConfig']:
         config['kubernetesMasterConfig']['proxyClientInfo'] = {
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index eea147229..0309e8a77 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -36,9 +36,9 @@
 
   - fail:
       msg: >
-        This upgrade is only supported for origin and openshift-enterprise
+        This upgrade is only supported for origin, openshift-enterprise, and online
         deployment types
-    when: deployment_type not in ['origin','openshift-enterprise']
+    when: deployment_type not in ['origin','openshift-enterprise', 'online']
 
   - fail:
       msg: >
@@ -517,24 +517,28 @@
     - _default_router.rc == 0
     - "'false' in _scc.stdout"
     command: >
-      {{ oc_cmd }} patch scc/privileged -p '{"allowHostPorts":true,"allowHostNetwork":true}' --loglevel=9
+      {{ oc_cmd }} patch scc/privileged -p
+      '{"allowHostPorts":true,"allowHostNetwork":true}' --api-version=v1
 
   - name: Update deployment config to 1.0.4/3.0.1 spec
     when: _default_router.rc == 0
     command: >
       {{ oc_cmd }} patch dc/router -p
       '{"spec":{"strategy":{"rollingParams":{"updatePercent":-10},"spec":{"serviceAccount":"router","serviceAccountName":"router"}}}}'
+      --api-version=v1
 
   - name: Switch to hostNetwork=true
     when: _default_router.rc == 0
     command: >
       {{ oc_cmd }} patch dc/router -p '{"spec":{"template":{"spec":{"hostNetwork":true}}}}'
+      --api-version=v1
 
   - name: Update router image to current version
     when: _default_router.rc == 0
     command: >
       {{ oc_cmd }} patch dc/router -p
       '{"spec":{"template":{"spec":{"containers":[{"name":"router","image":"{{ router_image }}"}]}}}}'
+      --api-version=v1
 
   - name: Check for default registry
     command: >
@@ -548,3 +552,4 @@
     command: >
       {{ oc_cmd }} patch dc/docker-registry -p
       '{"spec":{"template":{"spec":{"containers":[{"name":"registry","image":"{{ registry_image }}"}]}}}}'
+      --api-version=v1
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index ed23ada88..7d94ced2e 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -24,7 +24,7 @@
     - /etc/etcd/ca.crt
     register: g_etcd_server_cert_stat_result
   - set_fact:
-      etcd_server_certs_missing: "{{ g_etcd_server_cert_stat_result.results | map(attribute='stat.exists')
+      etcd_server_certs_missing: "{{ g_etcd_server_cert_stat_result.results | oo_collect(attribute='stat.exists')
                                     | list | intersect([false])}}"
       etcd_cert_subdir: etcd-{{ openshift.common.hostname }}
       etcd_cert_config_dir: /etc/etcd
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index b1da85d5d..196cdc8fe 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -60,7 +60,7 @@
     register: g_external_etcd_cert_stat_result
   - set_fact:
       etcd_client_certs_missing: "{{ g_external_etcd_cert_stat_result.results
-                                    | map(attribute='stat.exists')
+                                    | oo_collect(attribute='stat.exists')
                                     | list | intersect([false])}}"
       etcd_cert_subdir: openshift-master-{{ openshift.common.hostname }}
       etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
@@ -157,7 +157,7 @@
     register: g_master_cert_stat_result
   - set_fact:
       master_certs_missing: "{{ False in (g_master_cert_stat_result.results
-                                | map(attribute='stat.exists')
+                                | oo_collect(attribute='stat.exists')
                                 | list ) }}"
       master_cert_subdir: master-{{ openshift.common.hostname }}
       master_cert_config_dir: "{{ openshift.common.config_base }}/master"
@@ -204,14 +204,6 @@
       validate_checksum: yes
     with_items: masters_needing_certs
 
-- name: Inspect named certificates
-  hosts: oo_first_master
-  tasks:
-  - name: Collect certificate names
-    set_fact:
-      parsed_named_certificates: "{{ openshift_master_named_certificates | oo_parse_certificate_names(master_cert_config_dir, openshift.common.internal_hostnames) }}"
-    when: openshift_master_named_certificates is defined
-
 - name: Compute haproxy_backend_servers
   hosts: localhost
   connection: local
@@ -252,31 +244,69 @@
   - fail:
       msg: "openshift_master_session_auth_secrets and openshift_master_encryption_secrets must be equal length"
     when: (openshift_master_session_auth_secrets is defined and openshift_master_session_encryption_secrets is defined) and (openshift_master_session_auth_secrets | length != openshift_master_session_encryption_secrets | length)
+  - name: Install OpenSSL package
+    action: "{{ansible_pkg_mgr}} pkg=openssl state=present"
   - name: Generate session authentication key
     command: /usr/bin/openssl rand -base64 24
     register: session_auth_output
-    with_sequence: count=1
     when: openshift_master_session_auth_secrets is undefined
   - name: Generate session encryption key
     command: /usr/bin/openssl rand -base64 24
     register: session_encryption_output
-    with_sequence: count=1
     when: openshift_master_session_encryption_secrets is undefined
   - set_fact:
-      session_auth_secret: "{{ openshift_master_session_auth_secrets
-                                | default(session_auth_output.results
-                                | map(attribute='stdout')
-                                | list) }}"
-      session_encryption_secret: "{{ openshift_master_session_encryption_secrets
-                                      | default(session_encryption_output.results
-                                      | map(attribute='stdout')
-                                      | list) }}"
+      session_auth_secret: "{{ openshift_master_session_auth_secrets | default([session_auth_output.stdout]) }}"
+      session_encryption_secret: "{{ openshift_master_session_encryption_secrets | default([session_encryption_output.stdout]) }}"
+
+- name: Parse named certificates
+  hosts: localhost
+  vars:
+    internal_hostnames: "{{ hostvars[groups.oo_first_master.0].openshift.common.internal_hostnames }}"
+    named_certificates: "{{ hostvars[groups.oo_first_master.0].openshift_master_named_certificates | default([]) }}"
+    named_certificates_dir: "{{ hostvars[groups.oo_first_master.0].master_cert_config_dir }}/named_certificates/"
+  tasks:
+  - set_fact:
+      parsed_named_certificates: "{{ named_certificates | oo_parse_named_certificates(named_certificates_dir, internal_hostnames) }}"
+    when: named_certificates | length > 0
+
+- name: Deploy named certificates
+  hosts: oo_masters_to_config
+  vars:
+    named_certs_dir: "{{ master_cert_config_dir }}/named_certificates/"
+    named_certs_specified: "{{ openshift_master_named_certificates is defined }}"
+    overwrite_named_certs: "{{ openshift_master_overwrite_named_certificates | default(false) }}"
+  roles:
+  - role: openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        named_certificates: "{{ hostvars.localhost.parsed_named_certificates | default([]) }}"
+      additive_facts_to_overwrite:
+      - "{{ 'master.named_certificates' if overwrite_named_certs | bool else omit }}"
+  - name: Clear named certificates
+    file:
+      path: "{{ named_certs_dir }}"
+      state: absent
+    when: overwrite_named_certs | bool
+  - name: Ensure named certificate directory exists
+    file:
+      path: "{{ named_certs_dir }}"
+      state: directory
+    when: named_certs_specified | bool
+  - name: Land named certificates
+    copy: src="{{ item.certfile }}" dest="{{ named_certs_dir }}"
+    with_items: openshift_master_named_certificates
+    when: named_certs_specified | bool
+  - name: Land named certificate keys
+    copy: src="{{ item.keyfile }}" dest="{{ named_certs_dir }}"
+    with_items: openshift_master_named_certificates
+    when: named_certs_specified | bool
 
 - name: Configure master instances
   hosts: oo_masters_to_config
   serial: 1
   vars:
-    named_certificates: "{{ hostvars[groups['oo_first_master'][0]]['parsed_named_certificates'] | default([])}}"
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
     openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
     openshift_master_count: "{{ groups.oo_masters_to_config | length }}"
@@ -314,20 +344,8 @@
   - openshift_examples
   - role: openshift_cluster_metrics
     when: openshift.common.use_cluster_metrics | bool
-
-- name: Determine cluster dns ip
-  hosts: oo_first_master
-  tasks:
-  - name: Get master service ip
-    command: "{{ openshift.common.client_binary }} get -o template svc kubernetes --template=\\{\\{.spec.clusterIP\\}\\}"
-    register: master_service_ip_output
-    when: openshift.common.version_greater_than_3_1_or_1_1 | bool
-  - set_fact:
-      cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
-    when: not openshift.common.version_greater_than_3_1_or_1_1 | bool
-  - set_fact:
-      cluster_dns_ip: "{{ master_service_ip_output.stdout }}"
-    when: openshift.common.version_greater_than_3_1_or_1_1 | bool
+  - role: openshift_manageiq
+    when: openshift.common.use_manageiq | bool
 
 - name: Enable cockpit
   hosts: oo_first_master
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 8da9e231f..952a9fd51 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -33,7 +33,7 @@
     - server.crt
     register: stat_result
   - set_fact:
-      certs_missing: "{{ stat_result.results | map(attribute='stat.exists')
+      certs_missing: "{{ stat_result.results | oo_collect(attribute='stat.exists')
                          | list | intersect([false])}}"
       node_subdir: node-{{ openshift.common.hostname }}
       config_dir: "{{ openshift.common.config_base }}/generated-configs/node-{{ openshift.common.hostname }}"
@@ -48,7 +48,7 @@
     when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config and (openshift.common.use_flannel | bool)
   - set_fact:
       etcd_client_flannel_certs_missing: "{{ g_external_etcd_flannel_cert_stat_result.results
-                                             | map(attribute='stat.exists')
+                                             | oo_collect(attribute='stat.exists')
                                              | list | intersect([false])}}"
       etcd_cert_subdir: openshift-node-{{ openshift.common.hostname }}
       etcd_cert_config_dir: "{{ openshift.common.config_base }}/node"
@@ -158,8 +158,10 @@
   vars:
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
     openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
+    # TODO: Prefix flannel role variables.
     etcd_urls: "{{ hostvars[groups.oo_first_master.0].openshift.master.etcd_urls }}"
     embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}"
+    openshift_node_first_master_ip: "{{ hostvars[groups.oo_first_master.0].openshift.common.ip }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
diff --git a/playbooks/gce/openshift-cluster/join_node.yml b/playbooks/gce/openshift-cluster/join_node.yml
index c8f6065cd..5ae3a8fef 100644
--- a/playbooks/gce/openshift-cluster/join_node.yml
+++ b/playbooks/gce/openshift-cluster/join_node.yml
@@ -45,5 +45,3 @@
     openshift_use_openshift_sdn: true
     openshift_node_labels: "{{ lookup('oo_option', 'openshift_node_labels') }} "
     os_sdn_network_plugin_name: "redhat/openshift-ovs-subnet"
-    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
-    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].cluster_dns_ip }}"
diff --git a/playbooks/gce/openshift-cluster/launch.yml b/playbooks/gce/openshift-cluster/launch.yml
index 8be5d53e7..d6ef57c45 100644
--- a/playbooks/gce/openshift-cluster/launch.yml
+++ b/playbooks/gce/openshift-cluster/launch.yml
@@ -16,6 +16,8 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
       g_sub_host_type: "default"
+      gce_machine_type: "{{ lookup('env', 'gce_machine_master_type') | default(lookup('env', 'gce_machine_type'), true) }}"
+      gce_machine_image: "{{ lookup('env', 'gce_machine_master_image') | default(lookup('env', 'gce_machine_image'), true) }}"
 
   - include: ../../common/openshift-cluster/tasks/set_node_launch_facts.yml
     vars:
@@ -27,6 +29,8 @@
       cluster: "{{ cluster_id }}"
       type: "{{ k8s_type }}"
       g_sub_host_type: "{{ sub_host_type }}"
+      gce_machine_type: "{{ lookup('env', 'gce_machine_node_type') | default(lookup('env', 'gce_machine_type'), true) }}"
+      gce_machine_image: "{{ lookup('env', 'gce_machine_node_image') | default(lookup('env', 'gce_machine_image'), true) }}"
 
   - include: ../../common/openshift-cluster/tasks/set_node_launch_facts.yml
     vars:
diff --git a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
index c428cb465..de8a75b18 100644
--- a/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/gce/openshift-cluster/tasks/launch_instances.yml
@@ -5,8 +5,8 @@
 - name: Launch instance(s)
   gce:
     instance_names: "{{ instances }}"
-    machine_type: "{{ lookup('env', 'gce_machine_type') | default('n1-standard-1', true) }}"
-    image: "{{ lookup('env', 'gce_machine_image') | default(deployment_vars[deployment_type].image, true) }}"
+    machine_type: "{{ gce_machine_type | default(deployment_vars[deployment_type].machine_type, true) }}"
+    image: "{{ gce_machine_image | default(deployment_vars[deployment_type].image, true) }}"
     service_account_email: "{{ lookup('env', 'gce_service_account_email_address') }}"
     pem_file: "{{ lookup('env', 'gce_service_account_pem_file_path') }}"
     project_id: "{{ lookup('env', 'gce_project_id') }}"
diff --git a/playbooks/gce/openshift-cluster/vars.yml b/playbooks/gce/openshift-cluster/vars.yml
index 6de007807..a8ce8eb22 100644
--- a/playbooks/gce/openshift-cluster/vars.yml
+++ b/playbooks/gce/openshift-cluster/vars.yml
@@ -5,13 +5,16 @@ sdn_network_plugin: redhat/openshift-ovs-subnet
 deployment_vars:
   origin:
     image: preinstalled-slave-50g-v5
+    machine_type: n1-standard-1
     ssh_user: root
     sudo: yes
   online:
     image: libra-rhel7
+    machine_type: n1-standard-1
     ssh_user: root
     sudo: no
   enterprise:
     image: rhel-7
+    machine_type: n1-standard-1
     ssh_user:
     sudo: yes
diff --git a/playbooks/openstack/openshift-cluster/files/heat_stack.yaml b/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
index 40e4ab22c..bfd73c777 100644
--- a/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
+++ b/playbooks/openstack/openshift-cluster/files/heat_stack.yaml
@@ -43,6 +43,11 @@ parameters:
     description: Source of legitimate ssh connections
     default: 0.0.0.0/0
 
+  num_etcd:
+    type: number
+    label: Number of etcd nodes
+    description: Number of etcd nodes
+
   num_masters:
     type: number
     label: Number of masters
@@ -58,6 +63,11 @@ parameters:
     label: Number of infrastructure nodes
     description: Number of infrastructure nodes
 
+  etcd_image:
+    type: string
+    label: Etcd image
+    description: Name of the image for the etcd servers
+
   master_image:
     type: string
     label: Master image
@@ -73,6 +83,11 @@ parameters:
     label: Infra image
     description: Name of the image for the infra node servers
 
+  etcd_flavor:
+    type: string
+    label: Etcd flavor
+    description: Flavor of the etcd servers
+
   master_flavor:
     type: string
     label: Master flavor
@@ -90,6 +105,18 @@ parameters:
 
 outputs:
 
+  etcd_names:
+    description: Name of the etcds
+    value: { get_attr: [ etcd, name ] }
+
+  etcd_ips:
+    description: IPs of the etcds
+    value: { get_attr: [ etcd, private_ip ] }
+
+  etcd_floating_ips:
+    description: Floating IPs of the etcds
+    value: { get_attr: [ etcd, floating_ip ] }
+
   master_names:
     description: Name of the masters
     value: { get_attr: [ masters, name ] }
@@ -220,6 +247,37 @@ resources:
           port_range_min: 24224
           port_range_max: 24224
 
+  etcd-secgrp:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name:
+        str_replace:
+          template: openshift-ansible-cluster_id-etcd-secgrp
+          params:
+            cluster_id: { get_param: cluster_id }
+      description:
+        str_replace:
+          template: Security group for cluster_id etcd cluster
+          params:
+            cluster_id: { get_param: cluster_id }
+      rules:
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 22
+          port_range_max: 22
+          remote_ip_prefix: { get_param: ssh_incoming }
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 2379
+          port_range_max: 2379
+          remote_mode: remote_group_id
+          remote_group_id: { get_resource: master-secgrp }
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 2380
+          port_range_max: 2380
+          remote_mode: remote_group_id
+
   node-secgrp:
     type: OS::Neutron::SecurityGroup
     properties:
@@ -274,6 +332,36 @@ resources:
           port_range_min: 443
           port_range_max: 443
 
+  etcd:
+    type: OS::Heat::ResourceGroup
+    properties:
+      count: { get_param: num_etcd }
+      resource_def:
+        type: heat_stack_server.yaml
+        properties:
+          name:
+            str_replace:
+              template: cluster_id-k8s_type-%index%
+              params:
+                cluster_id: { get_param: cluster_id }
+                k8s_type: etcd
+          cluster_id: { get_param: cluster_id }
+          type:       etcd
+          image:      { get_param: etcd_image }
+          flavor:     { get_param: etcd_flavor }
+          key_name:   { get_resource: keypair }
+          net:        { get_resource: net }
+          subnet:     { get_resource: subnet }
+          secgrp:
+            - { get_resource: etcd-secgrp }
+          floating_network: { get_param: floating_ip_pool }
+          net_name:
+            str_replace:
+              template: openshift-ansible-cluster_id-net
+              params:
+                cluster_id: { get_param: cluster_id }
+    depends_on: interface
+
   masters:
     type: OS::Heat::ResourceGroup
     properties:
diff --git a/playbooks/openstack/openshift-cluster/launch.yml b/playbooks/openstack/openshift-cluster/launch.yml
index 651aef40b..b18512495 100644
--- a/playbooks/openstack/openshift-cluster/launch.yml
+++ b/playbooks/openstack/openshift-cluster/launch.yml
@@ -35,12 +35,15 @@
              -P floating_ip_pool={{ openstack_floating_ip_pool }}
              -P ssh_public_key="{{ openstack_ssh_public_key }}"
              -P ssh_incoming={{ openstack_ssh_access_from }}
+             -P num_etcd={{ num_etcd }}
              -P num_masters={{ num_masters }}
              -P num_nodes={{ num_nodes }}
              -P num_infra={{ num_infra }}
+             -P etcd_image={{ deployment_vars[deployment_type].image }}
              -P master_image={{ deployment_vars[deployment_type].image }}
              -P node_image={{ deployment_vars[deployment_type].image }}
              -P infra_image={{ deployment_vars[deployment_type].image }}
+             -P etcd_flavor={{ openstack_flavor["etcd"] }}
              -P master_flavor={{ openstack_flavor["master"] }}
              -P node_flavor={{ openstack_flavor["node"] }}
              -P infra_flavor={{ openstack_flavor["infra"] }}
@@ -61,6 +64,18 @@
   - set_fact:
       parsed_outputs: "{{ stack_show_result | oo_parse_heat_stack_outputs }}"
 
+  - name: Add new etcd instances groups and variables
+    add_host:
+      hostname: '{{ item[0] }}'
+      ansible_ssh_host: '{{ item[2] }}'
+      ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+      ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
+      groups: 'tag_env_{{ cluster_id }}, tag_host-type_etcd, tag_env-host-type_{{ cluster_id }}-openshift-etcd, tag_sub-host-type_default'
+    with_together:
+      - parsed_outputs.etcd_names
+      - parsed_outputs.etcd_ips
+      - parsed_outputs.etcd_floating_ips
+
   - name: Add new master instances groups and variables
     add_host:
       hostname: '{{ item[0] }}'
diff --git a/playbooks/openstack/openshift-cluster/vars.yml b/playbooks/openstack/openshift-cluster/vars.yml
index 262d3f4ed..e3796c91f 100644
--- a/playbooks/openstack/openshift-cluster/vars.yml
+++ b/playbooks/openstack/openshift-cluster/vars.yml
@@ -14,6 +14,7 @@ openstack_ssh_public_key:       "{{ lookup('file', lookup('oo_option', 'public_k
 openstack_ssh_access_from:      "{{ lookup('oo_option', 'ssh_from')          |
                                     default('0.0.0.0/0',                     True) }}"
 openstack_flavor:
+  etcd:   "{{ lookup('oo_option', 'etcd_flavor'      ) | default('m1.small',  True) }}"
   master: "{{ lookup('oo_option', 'master_flavor'    ) | default('m1.small',  True) }}"
   infra:  "{{ lookup('oo_option', 'infra_flavor'     ) | default('m1.small',  True) }}"
   node:   "{{ lookup('oo_option', 'node_flavor'      ) | default('m1.medium', True) }}"