16 files changed, 93 insertions, 128 deletions
diff --git a/playbooks/aws/README.md b/playbooks/aws/README.md
index c1793c505..0fb29ca06 100644
--- a/playbooks/aws/README.md
+++ b/playbooks/aws/README.md
@@ -34,6 +34,7 @@ Before any provisioning may occur, AWS account credentials must be present in th
 The newly added playbooks are the following:
 - build_ami.yml
 - provision.yml
+- provision_nodes.yml
 
 The current expected work flow should be to provide the `vars.yml` file with the
 desired settings for cluster instances.  These settings are AWS specific and should
@@ -52,19 +53,6 @@ provision:
     # when creating an encrypted AMI please specify use_encryption
     use_encryption: False
 
-    yum_repositories: # this is an example repository but it requires sslclient info.  Use a valid yum repository for openshift rpms
-    - name: openshift-repo
-      file: openshift-repo
-      description: OpenShift Builds
-      baseurl: https://mirror.openshift.com/enterprise/online-int/latest/x86_64/os/
-      enabled: yes
-      gpgcheck: no
-      sslverify: no
-      # client cert and key required for this repository
-      sslclientcert: "/var/lib/yum/client-cert.pem"
-      sslclientkey: "/var/lib/yum/client-key.pem"
-      gpgkey: "https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted"
-
   # for s3 registry backend
   openshift_registry_s3: True
 
@@ -123,40 +111,11 @@ provision:
 ```
 
 Repeat the following setup for the infra and compute node groups.  This most likely
- will not need editing but if further customization is required these parameters
+ will not need editing but if the install requires further customization then these parameters
  can be updated.
 
 #### Step 1
 
-Once the vars.yml file has been updated with the correct settings for the desired AWS account then we are ready to build an AMI.
-
-```
-$ ansible-playbook build_ami.yml
-```
-
-1. This script will build a VPC. Default name will be clusterid if not specified.
-2. Create an ssh key required for the instance.
-3. Create an instance.
-4. Run some setup roles to ensure packages and services are correctly configured.
-5. Create the AMI.
-6. If encryption is desired
-  - A KMS key is created with the name of $clusterid
-  - An encrypted AMI will be produced with $clusterid KMS key
-7. Terminate the instance used to configure the AMI.
-
-#### Step 2
-
-Now that we have created an AMI for our Openshift installation, that AMI id needs to be placed in the `vars.yml` file.  To do so update the following fields (The AMI can be captured from the output of the previous step or found in the ec2 console under AMIs):
-
-```
-  # when creating an encrypted AMI please specify use_encryption
-  use_encryption: False # defaults to false
-```
-
-**Note**: If using encryption, specify with `use_encryption: True`.  This will ensure to take the recently created AMI and encrypt it to be used later.  If encryption is not desired then set the value to false. The AMI id will be fetched and used according to its most recent creation date. 
-
-#### Step 3
-
 Create an openshift-ansible inventory file to use for a byo installation.  The exception here is that there will be no hosts specified by the inventory file.  Here is an example:
 
 ```ini
@@ -171,10 +130,20 @@ nodes
 etcd
 
 [OSEv3:vars]
-# cluster specific settings maybe be placed here
+################################################################################
+# Ensure these variables are set for bootstrap
+################################################################################
+openshift_master_bootstrap_enabled=True
+
 openshift_hosted_router_wait=False
 openshift_hosted_registry_wait=False
 
+# Repository for installation
+openshift_additional_repos=[{'name': 'openshift-repo', 'id': 'openshift-repo',  'baseurl': 'https://mirror.openshift.com/enterprise/enterprise-3.6/latest/x86_64/os/', 'enabled': 'yes', 'gpgcheck': 0, 'sslverify': 'no', 'sslclientcert': '/var/lib/yum/client-cert.pem', 'sslclientkey': '/var/lib/yum/client-key.pem', 'gpgkey': 'https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted'}]
+
+################################################################################
+# cluster specific settings maybe be placed here
+
 [masters]
 
 [etcd]
@@ -184,12 +153,46 @@ openshift_hosted_registry_wait=False
 
 There are more examples of cluster inventory settings [`here`](../../inventory/byo/).
 
+In order to create the bootstrapable AMI we need to create an openshift-ansible inventory file.  This file enables us to create the AMI using the openshift-ansible node roles.
+
+
+#### Step 2
+
+Once the vars.yml file has been updated with the correct settings for the desired AWS account then we are ready to build an AMI.
+
+```
+$ ansible-playbook -i inventory.yml build_ami.yml
+```
+
+1. This script will build a VPC. Default name will be clusterid if not specified.
+2. Create an ssh key required for the instance.
+3. Create an instance.
+4. Run some setup roles to ensure packages and services are correctly configured.
+5. Create the AMI.
+6. If encryption is desired
+  - A KMS key is created with the name of $clusterid
+  - An encrypted AMI will be produced with $clusterid KMS key
+7. Terminate the instance used to configure the AMI.
+
+
+#### Step 3
+
+Now that we have created an AMI for our Openshift installation, that AMI id needs to be placed in the `vars.yml` file.  To do so update the following fields (The AMI can be captured from the output of the previous step or found in the ec2 console under AMIs):
+
+```
+  # when creating an encrypted AMI please specify use_encryption
+  use_encryption: False # defaults to false
+```
+
+**Note**: If using encryption, specify with `use_encryption: True`.  This will ensure to take the recently created AMI and encrypt it to be used later.  If encryption is not desired then set the value to false. The AMI id will be fetched and used according to its most recent creation date.
+
+
 #### Step 4
 
 We are ready to create the master instances and install Openshift.
 
 ```
-$ ansible-playbook -i <inventory from step 3> provision.yml
+$ ansible-playbook -i <inventory from step 1> provision.yml
 ```
 
 This playbook runs through the following steps:
diff --git a/playbooks/aws/openshift-cluster/build_ami.yml b/playbooks/aws/openshift-cluster/build_ami.yml
index fa708ffa1..d27874200 100644
--- a/playbooks/aws/openshift-cluster/build_ami.yml
+++ b/playbooks/aws/openshift-cluster/build_ami.yml
@@ -60,24 +60,39 @@
       timeout: 300
       search_regex: OpenSSH
 
-  - name: add host to group
+  - name: add host to nodes
     add_host:
+      groups: nodes
       name: "{{ amibase.tagged_instances.0.public_dns_name }}"
-      groups: amibase
 
-- hosts: amibase
+  - name: set the user to perform installation
+    set_fact:
+      ansible_ssh_user: root
+
+- name: normalize groups
+  include: ../../byo/openshift-cluster/initialize_groups.yml
+
+- name: run the std_include
+  include: ../../common/openshift-cluster/evaluate_groups.yml
+
+- name: run the std_include
+  include: ../../common/openshift-cluster/initialize_facts.yml
+
+- name: run the std_include
+  include: ../../common/openshift-cluster/initialize_openshift_repos.yml
+
+- hosts: nodes
   remote_user: root
   tasks:
-  - name: included required variables
+  - name: get the necessary vars for ami building
     include_vars: vars.yml
 
+  - set_fact:
+      openshift_node_bootstrap: True
+
   - name: run openshift image preparation
     include_role:
-      name: openshift_ami_prep
-    vars:
-      r_openshift_ami_prep_yum_repositories: "{{ provision.build.yum_repositories }}"
-      r_openshift_ami_prep_node: atomic-openshift-node
-      r_openshift_ami_prep_master: atomic-openshift-master
+      name: openshift_node
 
 - hosts: localhost
   connection: local
@@ -90,6 +105,7 @@
       state: present
       description: "This was provisioned {{ ansible_date_time.iso8601 }}"
       name: "{{ provision.build.ami_name }}{{ lookup('pipe', 'date +%Y%m%d%H%M')}}"
+      tags: "{{ provision.build.openshift_ami_tags }}"
       wait: yes
     register: amioutput
 
diff --git a/playbooks/aws/openshift-cluster/vars.yml b/playbooks/aws/openshift-cluster/vars.yml
index 7810157d4..47da03cb7 100644
--- a/playbooks/aws/openshift-cluster/vars.yml
+++ b/playbooks/aws/openshift-cluster/vars.yml
@@ -10,17 +10,6 @@ provision:
   build:  # build specific variables here
     ami_name: "openshift-gi-"
     base_image: ami-bdd5d6ab  # base image for AMI to build from
-    yum_repositories:  # this is an example repository but it requires sslclient info
-    - name: openshift-repo
-      file: openshift-repo
-      description: OpenShift Builds
-      baseurl: https://mirror.openshift.com/enterprise/online-int/latest/x86_64/os/
-      enabled: yes
-      gpgcheck: no
-      sslverify: no
-      sslclientcert: "/var/lib/yum/client-cert.pem"
-      sslclientkey: "/var/lib/yum/client-key.pem"
-      gpgkey: "https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-release https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-beta https://mirror.ops.rhcloud.com/libra/keys/RPM-GPG-KEY-redhat-openshifthosted"
 
     # when creating an encrypted AMI please specify use_encryption
     use_encryption: False
diff --git a/playbooks/byo/openshift-master/scaleup.yml b/playbooks/byo/openshift-master/scaleup.yml
index 64811e80d..e3ef704e5 100644
--- a/playbooks/byo/openshift-master/scaleup.yml
+++ b/playbooks/byo/openshift-master/scaleup.yml
@@ -15,6 +15,8 @@
     when:
     - (g_new_master_hosts | default([]) | length == 0) or (g_new_node_hosts | default([]) | length == 0)
 
+- include: ../../common/openshift-cluster/std_include.yml
+
 - include: ../../common/openshift-master/scaleup.yml
   vars:
     openshift_cluster_id: "{{ cluster_id | default('default') }}"
diff --git a/playbooks/byo/openshift-node/scaleup.yml b/playbooks/byo/openshift-node/scaleup.yml
index fda89b1ea..0225623c6 100644
--- a/playbooks/byo/openshift-node/scaleup.yml
+++ b/playbooks/byo/openshift-node/scaleup.yml
@@ -14,6 +14,8 @@
     when:
     - g_new_node_hosts | default([]) | length == 0
 
+- include: ../../common/openshift-cluster/std_include.yml
+
 - include: ../../common/openshift-node/scaleup.yml
   vars:
     openshift_cluster_id: "{{ cluster_id | default('default') }}"
diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index e1df71112..26b27ba39 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -18,10 +18,6 @@
       - docker_image_availability
       - docker_storage
 
-- include: initialize_firewall.yml
-  tags:
-  - always
-
 - hosts: localhost
   tasks:
   - fail:
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index c56b07037..a1ae14a1f 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -13,12 +13,12 @@
   - name: Evaluate groups - g_master_hosts or g_new_master_hosts required
     fail:
       msg: This playbook requires g_master_hosts or g_new_master_hosts to be set
-    when: g_master_hosts is not defined or g_new_master_hosts is not defined
+    when: g_master_hosts is not defined and g_new_master_hosts is not defined
 
   - name: Evaluate groups - g_node_hosts or g_new_node_hosts required
     fail:
       msg: This playbook requires g_node_hosts or g_new_node_hosts to be set
-    when: g_node_hosts is not defined or g_new_node_hosts is not defined
+    when: g_node_hosts is not defined and g_new_node_hosts is not defined
 
   - name: Evaluate groups - g_lb_hosts required
     fail:
@@ -117,7 +117,7 @@
     add_host:
       name: "{{ item }}"
       groups: oo_etcd_hosts_to_backup
-    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}"
+    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else (groups.oo_first_master | default([])) }}"
     changed_when: False
 
   - name: Evaluate oo_nodes_to_config
@@ -173,5 +173,5 @@
       groups: oo_etcd_to_migrate
       ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
       ansible_become: "{{ g_sudo | default(omit) }}"
-    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config | default([]) | length != 0 else groups.oo_first_master }}"
+    with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config | default([]) | length != 0 else (groups.oo_first_master |default([]))}}"
     changed_when: no
diff --git a/playbooks/common/openshift-cluster/initialize_facts.yml b/playbooks/common/openshift-cluster/initialize_facts.yml
index 65be436c6..9eaf3bc34 100644
--- a/playbooks/common/openshift-cluster/initialize_facts.yml
+++ b/playbooks/common/openshift-cluster/initialize_facts.yml
@@ -157,4 +157,4 @@
 
   - name: initialize_facts set_fact on openshift_docker_hosted_registry_network
     set_fact:
-      openshift_docker_hosted_registry_network: "{{ hostvars[groups.oo_first_master.0].openshift.common.portal_net }}"
+      openshift_docker_hosted_registry_network: "{{ '' if 'oo_first_master' not in groups else hostvars[groups.oo_first_master.0].openshift.common.portal_net }}"
diff --git a/playbooks/common/openshift-cluster/initialize_firewall.yml b/playbooks/common/openshift-cluster/initialize_firewall.yml
deleted file mode 100644
index f0374fbc7..000000000
--- a/playbooks/common/openshift-cluster/initialize_firewall.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-- name: Initialize host firewall
-  hosts: oo_all_hosts
-  tasks:
-  - name: Install and configure the proper firewall settings
-    include_role:
-      name: os_firewall
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 2cb6197d1..f2b85eea1 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -3,6 +3,7 @@
   hosts: oo_etcd_to_config
   any_errors_fatal: true
   roles:
+  - role: os_firewall
   - role: openshift_etcd
     etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
     etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
diff --git a/playbooks/common/openshift-etcd/scaleup.yml b/playbooks/common/openshift-etcd/scaleup.yml
index 52b90daca..5f8bb1c7a 100644
--- a/playbooks/common/openshift-etcd/scaleup.yml
+++ b/playbooks/common/openshift-etcd/scaleup.yml
@@ -28,13 +28,15 @@
     delay: 10
     until: etcd_add_check.rc == 0
   roles:
+  - role: os_firewall
+    when: etcd_add_check.rc == 0
   - role: openshift_etcd
     when: etcd_add_check.rc == 0
     etcd_peers: "{{ groups.oo_etcd_to_config | union(groups.oo_new_etcd_to_config)| default([], true) }}"
     etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
     etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
     etcd_initial_cluster_state: "existing"
-    initial_etcd_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') }}"
+    initial_etcd_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}"
     etcd_ca_setup: False
     r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   - role: nickhammond.logrotate
@@ -47,5 +49,7 @@
                        --ca-file {{ etcd_peer_ca_file }}
                        -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }}
                        cluster-health
-    retries: 1
+    register: scaleup_health
+    retries: 3
     delay: 30
+    until: scaleup_health.rc == 0
diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml
index 2dacc1218..09ed81a83 100644
--- a/playbooks/common/openshift-loadbalancer/config.yml
+++ b/playbooks/common/openshift-loadbalancer/config.yml
@@ -14,4 +14,5 @@
                                           + openshift_loadbalancer_additional_backends | default([]) }}"
     openshift_image_tag: "{{ hostvars[groups.oo_first_master.0].openshift_image_tag }}"
   roles:
+  - role: os_firewall
   - role: openshift_loadbalancer
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index b30450def..cd25dd211 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -180,6 +180,7 @@
                                                 | oo_collect('openshift.common.ip') | default([]) | join(',')
                                                 }}"
   roles:
+  - role: os_firewall
   - role: openshift_master
     openshift_ca_host: "{{ groups.oo_first_master.0 }}"
     openshift_master_etcd_hosts: "{{ hostvars
diff --git a/playbooks/common/openshift-master/scaleup.yml b/playbooks/common/openshift-master/scaleup.yml
index bc61ee9bb..6ad4cde65 100644
--- a/playbooks/common/openshift-master/scaleup.yml
+++ b/playbooks/common/openshift-master/scaleup.yml
@@ -1,11 +1,4 @@
 ---
-- include: ../openshift-cluster/evaluate_groups.yml
-
-- name: Gather facts
-  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
-  roles:
-  - openshift_facts
-
 - name: Update master count
   hosts: oo_masters:!oo_masters_to_config
   serial: 1
@@ -50,16 +43,6 @@
     delay: 1
     changed_when: false
 
-- name: Configure docker hosts
-  hosts: oo_masters_to_config:oo_nodes_to_config
-  vars:
-    docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}"
-    docker_insecure_registries: "{{ lookup('oo_option',  'docker_insecure_registries') | oo_split }}"
-    docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}"
-  roles:
-  - openshift_facts
-  - openshift_docker
-
 - name: Disable excluders
   hosts: oo_masters_to_config
   tags:
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index ef7d54f9f..04c811c22 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -32,6 +32,7 @@
                                                 }}"
 
   roles:
+  - role: os_firewall
   - role: openshift_node
     openshift_ca_host: "{{ groups.oo_first_master.0 }}"
 
@@ -47,6 +48,7 @@
                                                 | oo_collect('openshift.common.hostname') | default([]) | join (',')
                                                 }}"
   roles:
+  - role: os_firewall
   - role: openshift_node
     openshift_ca_host: "{{ groups.oo_first_master.0 }}"
 
diff --git a/playbooks/common/openshift-node/scaleup.yml b/playbooks/common/openshift-node/scaleup.yml
index 40da8990d..b1bbbb14c 100644
--- a/playbooks/common/openshift-node/scaleup.yml
+++ b/playbooks/common/openshift-node/scaleup.yml
@@ -1,32 +1,4 @@
 ---
-- include: ../openshift-cluster/evaluate_groups.yml
-
-- name: Gather facts
-  hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
-  roles:
-  - openshift_facts
-
-- name: Gather and set facts for first master
-  hosts: oo_first_master
-  vars:
-    openshift_master_count: "{{ groups.oo_masters | length }}"
-  pre_tasks:
-  - set_fact:
-      openshift_master_default_subdomain: "{{ lookup('oo_option', 'openshift_master_default_subdomain') | default(None, true) }}"
-    when: openshift_master_default_subdomain is not defined
-  roles:
-  - openshift_master_facts
-
-- name: Configure docker hosts
-  hosts: oo_nodes_to_config
-  vars:
-    docker_additional_registries: "{{ lookup('oo_option', 'docker_additional_registries') | oo_split }}"
-    docker_insecure_registries: "{{ lookup('oo_option',  'docker_insecure_registries') | oo_split }}"
-    docker_blocked_registries: "{{ lookup('oo_option', 'docker_blocked_registries') | oo_split }}"
-  roles:
-  - openshift_facts
-  - openshift_docker
-
 - name: Disable excluders
   hosts: oo_nodes_to_config
   tags: