Merge pull request #5361 from kwoodson/fix_bootstrap_files

Provisioning updates.

24 files changed, 429 insertions, 93 deletions

diff --git a/playbooks/aws/openshift-cluster/accept.yml b/playbooks/aws/openshift-cluster/accept.yml
index ffc367f9f..c2c8bea50 100755
--- a/playbooks/aws/openshift-cluster/accept.yml
+++ b/playbooks/aws/openshift-cluster/accept.yml
@@ -42,12 +42,12 @@
     until: "'instances' in instancesout and instancesout.instances|length > 0"
 
   - debug:
-      msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list | regex_replace('.ec2.internal') }}"
+      msg: "{{ instancesout.instances|map(attribute='private_dns_name') | list }}"
 
   - name: approve nodes
     oc_adm_csr:
       #approve_all: True
-      nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list | regex_replace('.ec2.internal') }}"
-      timeout: 0
+      nodes: "{{ instancesout.instances|map(attribute='private_dns_name') | list  }}"
+      timeout: 60
     register: nodeout
     delegate_to: "{{ mastersout.instances[0].public_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/build_ami.yml b/playbooks/aws/openshift-cluster/build_ami.yml
index 559a37cbe..ee281929a 100644
--- a/playbooks/aws/openshift-cluster/build_ami.yml
+++ b/playbooks/aws/openshift-cluster/build_ami.yml
@@ -27,6 +27,7 @@
   - name: set the user to perform installation
     set_fact:
       ansible_ssh_user: "{{ openshift_aws_build_ami_ssh_user | default('root') }}"
+      openshift_node_bootstrap: True
 
 # This is the part that installs all of the software and configs for the instance
 # to become a node.
diff --git a/playbooks/common/openshift-cluster/evaluate_groups.yml b/playbooks/common/openshift-cluster/evaluate_groups.yml
index e55b2f964..78b552279 100644
--- a/playbooks/common/openshift-cluster/evaluate_groups.yml
+++ b/playbooks/common/openshift-cluster/evaluate_groups.yml
@@ -51,7 +51,7 @@
     when:
     - g_etcd_hosts | default([]) | length not in [3,1]
     - not openshift_master_unsupported_embedded_etcd | default(False)
-    - not openshift_node_bootstrap | default(False)
+    - not (openshift_node_bootstrap | default(False))
 
   - name: Evaluate oo_all_hosts
     add_host:
diff --git a/playbooks/common/openshift-node/image_prep.yml b/playbooks/common/openshift-node/image_prep.yml
index fc06621ee..00d167c22 100644
--- a/playbooks/common/openshift-node/image_prep.yml
+++ b/playbooks/common/openshift-node/image_prep.yml
@@ -2,13 +2,13 @@
 - name: normalize groups
   include: ../../byo/openshift-cluster/initialize_groups.yml
 
-- name: run the std_include
+- name: evaluate the groups
   include: ../openshift-cluster/evaluate_groups.yml
 
-- name: run the std_include
+- name: initialize the facts
   include: ../openshift-cluster/initialize_facts.yml
 
-- name: run the std_include
+- name: initialize the repositories
   include: ../openshift-cluster/initialize_openshift_repos.yml
 
 - name: run node config setup
diff --git a/roles/lib_openshift/library/oc_adm_csr.py b/roles/lib_openshift/library/oc_adm_csr.py
index d1dc4caf8..324f52689 100644
--- a/roles/lib_openshift/library/oc_adm_csr.py
+++ b/roles/lib_openshift/library/oc_adm_csr.py
@@ -1478,11 +1478,23 @@ class OCcsr(OpenShiftCLI):
 
         return False
 
+    def get_csr_request(self, request):
+        '''base64 decode the request object and call openssl to determine the
+           subject and specifically the CN: from the request
+
+           Output:
+           (0, '...
+                Subject: O=system:nodes, CN=system:node:ip-172-31-54-54.ec2.internal
+                ...')
+        '''
+        import base64
+        return self._run(['openssl', 'req', '-noout', '-text'], base64.b64decode(request))[1]
+
     def match_node(self, csr):
         '''match an inc csr to a node in self.nodes'''
         for node in self.nodes:
-            # we have a match
-            if node['name'] in csr['metadata']['name']:
+            # we need to match based upon the csr's request certificate's CN
+            if node['name'] in self.get_csr_request(csr['spec']['request']):
                 node['csrs'][csr['metadata']['name']] = csr
 
                 # check that the username is the node and type is 'Approved'
diff --git a/roles/lib_openshift/src/class/oc_adm_csr.py b/roles/lib_openshift/src/class/oc_adm_csr.py
index ea11c6ca9..22b8f9165 100644
--- a/roles/lib_openshift/src/class/oc_adm_csr.py
+++ b/roles/lib_openshift/src/class/oc_adm_csr.py
@@ -66,11 +66,23 @@ class OCcsr(OpenShiftCLI):
 
         return False
 
+    def get_csr_request(self, request):
+        '''base64 decode the request object and call openssl to determine the
+           subject and specifically the CN: from the request
+
+           Output:
+           (0, '...
+                Subject: O=system:nodes, CN=system:node:ip-172-31-54-54.ec2.internal
+                ...')
+        '''
+        import base64
+        return self._run(['openssl', 'req', '-noout', '-text'], base64.b64decode(request))[1]
+
     def match_node(self, csr):
         '''match an inc csr to a node in self.nodes'''
         for node in self.nodes:
-            # we have a match
-            if node['name'] in csr['metadata']['name']:
+            # we need to match based upon the csr's request certificate's CN
+            if node['name'] in self.get_csr_request(csr['spec']['request']):
                 node['csrs'][csr['metadata']['name']] = csr
 
                 # check that the username is the node and type is 'Approved'
diff --git a/roles/openshift_aws/defaults/main.yml b/roles/openshift_aws/defaults/main.yml
index ea09857b0..5371588cf 100644
--- a/roles/openshift_aws/defaults/main.yml
+++ b/roles/openshift_aws/defaults/main.yml
@@ -4,7 +4,6 @@ openshift_aws_create_iam_cert: True
 openshift_aws_create_security_groups: True
 openshift_aws_create_launch_config: True
 openshift_aws_create_scale_group: True
-openshift_aws_kubernetes_cluster_status: owned  # or shared
 openshift_aws_node_group_type: master
 
 openshift_aws_wait_for_ssh: True
@@ -13,6 +12,7 @@ openshift_aws_clusterid: default
 openshift_aws_region: us-east-1
 openshift_aws_vpc_name: "{{ openshift_aws_clusterid }}"
 openshift_aws_build_ami_group: "{{ openshift_aws_clusterid }}"
+openshift_aws_kubernetes_cluster_status: "{{ openshift_aws_clusterid }}"
 
 openshift_aws_iam_cert_name: "{{ openshift_aws_clusterid }}-master-external"
 openshift_aws_iam_cert_path: ''
@@ -89,6 +89,10 @@ openshift_aws_node_group_config_node_volumes:
   delete_on_termination: True
 
 openshift_aws_node_group_config_tags: "{{ openshift_aws_clusterid | build_instance_tags(openshift_aws_kubernetes_cluster_status) }}"
+openshift_aws_node_group_termination_policy: Default
+openshift_aws_node_group_replace_instances: []
+openshift_aws_node_group_replace_all_instances: False
+openshift_aws_node_group_config_extra_labels: {}
 
 openshift_aws_node_group_config:
   tags: "{{ openshift_aws_node_group_config_tags }}"
@@ -105,7 +109,11 @@ openshift_aws_node_group_config:
     tags:
       host-type: master
       sub-host-type: default
+    labels:
+      type: master
     wait_for_instances: True
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   compute:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -119,6 +127,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: compute
+    labels:
+      type: compute
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
   infra:
     instance_type: m4.xlarge
     ami: "{{ openshift_aws_ami }}"
@@ -132,6 +144,10 @@ openshift_aws_node_group_config:
     tags:
       host-type: node
       sub-host-type: infra
+    labels:
+      type: infra
+    termination_policy: "{{ openshift_aws_node_group_termination_policy }}"
+    replace_all_instances: "{{ openshift_aws_node_group_replace_all_instances }}"
 
 openshift_aws_elb_security_groups:
 - "{{ openshift_aws_clusterid }}"
@@ -211,3 +227,7 @@ openshift_aws_vpc:
       az: "us-east-1e"
     - cidr: 172.31.16.0/20
       az: "us-east-1a"
+
+openshift_aws_node_run_bootstrap_startup: True
+openshift_aws_node_user_data: ''
+openshift_aws_node_config_namespace: openshift-node
diff --git a/roles/openshift_aws/tasks/launch_config.yml b/roles/openshift_aws/tasks/launch_config.yml
index e6be9969c..8b7b02a0e 100644
--- a/roles/openshift_aws/tasks/launch_config.yml
+++ b/roles/openshift_aws/tasks/launch_config.yml
@@ -4,6 +4,11 @@
   when:
   - openshift_aws_ami is undefined
 
+- fail:
+    msg: "Ensure that openshift_deployment_type is defined."
+  when:
+  - openshift_deployment_type is undefined
+
 - name: query vpc
   ec2_vpc_net_facts:
     region: "{{ openshift_aws_region }}"
@@ -27,23 +32,7 @@
     image_id: "{{ openshift_aws_ami }}"
     instance_type: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].instance_type }}"
     security_groups: "{{ openshift_aws_launch_config_security_group_id  | default(ec2sgs.security_groups | map(attribute='group_id')| list) }}"
-    user_data: |-
-      #cloud-config
-      {%  if openshift_aws_node_group_type != 'master' %}
-      write_files:
-      - path: /root/csr_kubeconfig
-        owner: root:root
-        permissions: '0640'
-        content: {{ openshift_aws_launch_config_bootstrap_token | default('') | to_yaml }}
-      - path: /root/openshift_settings
-        owner: root:root
-        permissions: '0640'
-        content:
-          openshift_type: "{{ openshift_aws_node_group_type }}"
-      runcmd:
-      - [ systemctl, enable, atomic-openshift-node]
-      - [ systemctl, start, atomic-openshift-node]
-      {% endif %}
+    user_data: "{{ lookup('template', 'user_data.j2') }}"
     key_name: "{{ openshift_aws_ssh_key_name }}"
     ebs_optimized: False
     volumes: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].volumes }}"
diff --git a/roles/openshift_aws/tasks/provision_instance.yml b/roles/openshift_aws/tasks/provision_instance.yml
index 1384bae59..25ae6ce1c 100644
--- a/roles/openshift_aws/tasks/provision_instance.yml
+++ b/roles/openshift_aws/tasks/provision_instance.yml
@@ -1,4 +1,8 @@
 ---
+- name: set openshift_node_bootstrap to True when building AMI
+  set_fact:
+    openshift_node_bootstrap: True
+
 - name: query vpc
   ec2_vpc_net_facts:
     region: "{{ openshift_aws_region }}"
@@ -53,10 +57,6 @@
     timeout: 300
     search_regex: OpenSSH
 
-- name: Pause 10 seconds to ensure ssh actually accepts logins
-  pause:
-    seconds: 20
-
 - name: add host to nodes
   add_host:
     groups: nodes
diff --git a/roles/openshift_aws/tasks/scale_group.yml b/roles/openshift_aws/tasks/scale_group.yml
index 3e969fc43..eb31636e7 100644
--- a/roles/openshift_aws/tasks/scale_group.yml
+++ b/roles/openshift_aws/tasks/scale_group.yml
@@ -28,5 +28,7 @@
     load_balancers: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].elbs if 'elbs' in openshift_aws_node_group_config[openshift_aws_node_group_type] else omit }}"
     wait_for_instances: "{{ openshift_aws_node_group_config[openshift_aws_node_group_type].wait_for_instances | default(False)}}"
     vpc_zone_identifier: "{{ subnetout.subnets[0].id }}"
+    replace_instances: "{{ openshift_aws_node_group_replace_instances if openshift_aws_node_group_replace_instances != [] else omit }}"
+    replace_all_instances: "{{ omit if openshift_aws_node_group_replace_instances != [] else (openshift_aws_node_group_config[openshift_aws_node_group_type].replace_all_instances | default(omit)) }}"
     tags:
     - "{{ openshift_aws_node_group_config.tags | combine(openshift_aws_node_group_config[openshift_aws_node_group_type].tags) }}"
diff --git a/roles/openshift_aws/tasks/seal_ami.yml b/roles/openshift_aws/tasks/seal_ami.yml
index 0cb749dcc..d319fdd1a 100644
--- a/roles/openshift_aws/tasks/seal_ami.yml
+++ b/roles/openshift_aws/tasks/seal_ami.yml
@@ -1,4 +1,11 @@
 ---
+- name: Remove any ansible facts created during AMI creation
+  file:
+    path: "/etc/ansible/facts.d/{{ item }}"
+    state: absent
+  with_items:
+  - openshift.fact
+
 - name: fetch newly created instances
   ec2_remote_facts:
     region: "{{ openshift_aws_region }}"
diff --git a/roles/openshift_aws/templates/user_data.j2 b/roles/openshift_aws/templates/user_data.j2
new file mode 100644
index 000000000..ed9c0ed0b
--- /dev/null
+++ b/roles/openshift_aws/templates/user_data.j2
@@ -0,0 +1,26 @@
+{% if openshift_aws_node_user_data is defined and openshift_aws_node_user_data != '' %}
+{{ openshift_aws_node_user_data }}
+{% else %}
+#cloud-config
+write_files:
+- path: /root/openshift_bootstrap/openshift_settings.yaml
+  owner: 'root:root'
+  permissions: '0640'
+  content: |
+    openshift_group_type: {{ openshift_aws_node_group_type }}
+{%   if openshift_aws_node_group_type != 'master' %}
+- path: /etc/origin/node/csr_kubeconfig
+  owner: 'root:root'
+  permissions: '0640'
+  encoding: b64
+  content: {{ openshift_aws_launch_config_bootstrap_token | b64encode }}
+{%   endif %}
+runcmd:
+{%     if openshift_aws_node_run_bootstrap_startup %}
+- [ ansible-playbook, /root/openshift_bootstrap/bootstrap.yml]
+{%     endif %}
+{%     if openshift_aws_node_group_type != 'master' %}
+- [ systemctl, enable, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node]
+- [ systemctl, start, {% if openshift_deployment_type == 'openshift-enterprise' %}atomic-openshift{% else %}origin{% endif %}-node]
+{%     endif %}
+{% endif %}
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index f861a8e4d..b6875ebd4 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -54,3 +54,88 @@ r_openshift_master_sdn_network_plugin_name: "{{ r_openshift_master_sdn_network_p
 
 openshift_master_image_config_latest_default: "{{ openshift_image_config_latest | default(False) }}"
 openshift_master_image_config_latest: "{{ openshift_master_image_config_latest_default }}"
+
+openshift_master_config_dir_default: "{{ (openshift.common.config_base | default('/etc/origin/master')) ~ '/master' }}"
+openshift_master_config_dir: "{{ openshift_master_config_dir_default }}"
+openshift_master_cloud_provider: "{{ openshift_cloudprovider_kind | default('aws') }}"
+
+openshift_master_node_config_networkconfig_mtu: 1450
+
+openshift_master_node_config_kubeletargs_cpu: 500m
+openshift_master_node_config_kubeletargs_mem: 512M
+
+openshift_master_bootstrap_enabled: False
+
+openshift_master_client_binary: "{{ openshift.common.client_binary if openshift is defined else 'oc' }}"
+
+openshift_master_config_imageconfig_format: "{{ oreg_url if oreg_url != '' else 'registry.access.redhat.com/openshift3/ose-${component}:${version}' }}"
+
+# these are for the default settings in a generated node-config.yaml
+openshift_master_node_config_default_edits:
+- key: nodeName
+  state: absent
+- key: dnsBindAddress
+  value: 127.0.0.1:53
+- key: dnsDomain
+  value: cluster.local
+- key: dnsRecursiveResolvConf
+  value: /etc/origin/node/resolv.conf
+- key: imageConfig.format
+  value: "{{ openshift_master_config_imageconfig_format }}"
+- key: kubeletArguments.cloud-config
+  value:
+  - "/etc/origin/cloudprovider/{{ openshift_master_cloud_provider }}.conf"
+- key: kubeletArguments.cloud-provider
+  value:
+  - "{{ openshift_master_cloud_provider }}"
+- key: kubeletArguments.kube-reserved
+  value:
+  - "cpu={{ openshift_master_node_config_kubeletargs_cpu }},memory={{ openshift_master_node_config_kubeletargs_mem }}"
+- key: kubeletArguments.system-reserved
+  value:
+  - "cpu={{ openshift_master_node_config_kubeletargs_cpu }},memory={{ openshift_master_node_config_kubeletargs_mem }}"
+- key: enable-controller-attach-detach
+  value:
+  - 'true'
+- key: networkConfig.mtu
+  value: 8951
+- key: networkConfig.networkPluginName
+  value: "{{ r_openshift_master_sdn_network_plugin_name }}"
+- key: networkPluginName
+  value: "{{ r_openshift_master_sdn_network_plugin_name }}"
+
+
+# We support labels for all nodes here
+openshift_master_node_config_kubeletargs_default_labels: []
+# We do support overrides for node group labels
+openshift_master_node_config_kubeletargs_master_labels: []
+openshift_master_node_config_kubeletargs_infra_labels: []
+openshift_master_node_config_kubeletargs_compute_labels: []
+
+openshift_master_node_config_master:
+  type: master
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_master_labels) |
+               union(['type=master']) }}"
+openshift_master_node_config_infra:
+  type: infra
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_infra_labels) |
+               union(['type=infra']) }}"
+openshift_master_node_config_compute:
+  type: compute
+  edits:
+  - key: kubeletArguments.node-labels
+    value: "{{ openshift_master_node_config_kubeletargs_default_labels |
+               union(openshift_master_node_config_kubeletargs_compute_labels) |
+               union(['type=compute']) }}"
+
+openshift_master_node_configs:
+- "{{ openshift_master_node_config_infra }}"
+- "{{ openshift_master_node_config_compute }}"
+
+openshift_master_bootstrap_namespace: openshift-node
diff --git a/roles/openshift_master/meta/main.yml b/roles/openshift_master/meta/main.yml
index a657668a9..a1cda2ad4 100644
--- a/roles/openshift_master/meta/main.yml
+++ b/roles/openshift_master/meta/main.yml
@@ -13,4 +13,5 @@ galaxy_info:
   - cloud
 dependencies:
 - role: lib_openshift
+- role: lib_utils
 - role: lib_os_firewall
diff --git a/roles/openshift_master/tasks/bootstrap.yml b/roles/openshift_master/tasks/bootstrap.yml
index 0013f5289..eee89743c 100644
--- a/roles/openshift_master/tasks/bootstrap.yml
+++ b/roles/openshift_master/tasks/bootstrap.yml
@@ -26,3 +26,66 @@
   copy:
     content: "{{ kubeconfig_out.stdout }}"
     dest: "{{ openshift_master_config_dir }}/bootstrap.kubeconfig"
+
+- name: create a temp dir for this work
+  command: mktemp -d /tmp/openshift_node_config-XXXXXX
+  register: mktempout
+  run_once: true
+
+# This generate is so that we do not have to maintain
+# our own copy of the template.  This is generated by
+# the product and the following settings will be
+# generated by the master
+- name: generate a node-config dynamically
+  command: >
+    {{ openshift_master_client_binary }} adm create-node-config
+    --node-dir={{ mktempout.stdout }}/
+    --node=CONFIGMAP
+    --hostnames=test
+    --certificate-authority={{ openshift_master_config_dir }}/ca.crt
+    --signer-cert={{ openshift_master_config_dir }}/ca.crt
+    --signer-key={{ openshift_master_config_dir }}/ca.key
+    --signer-serial={{ openshift_master_config_dir }}/ca.serial.txt
+    --node-client-certificate-authority={{ openshift_master_config_dir }}/ca.crt
+  register: configgen
+  run_once: true
+
+- name: remove the default settings
+  yedit:
+    state: "{{ item.state | default('present') }}"
+    src: "{{ mktempout.stdout }}/node-config.yaml"
+    key: "{{ item.key }}"
+    value: "{{ item.value | default(omit) }}"
+  with_items: "{{ openshift_master_node_config_default_edits }}"
+  run_once: true
+
+- name: copy the generated config into each group
+  copy:
+    src: "{{ mktempout.stdout }}/node-config.yaml"
+    remote_src: true
+    dest: "{{ mktempout.stdout }}/node-config-{{ item.type }}.yaml"
+  with_items: "{{ openshift_master_node_configs }}"
+  run_once: true
+
+- name: "specialize the generated configs for node-config-{{ item.type }}"
+  yedit:
+    src: "{{ mktempout.stdout }}/node-config-{{ item.type }}.yaml"
+    edits: "{{ item.edits }}"
+  with_items: "{{ openshift_master_node_configs }}"
+  run_once: true
+
+- name: create node-config.yaml configmap
+  oc_configmap:
+    name: "node-config-{{ item.type }}"
+    namespace: "{{ openshift_master_bootstrap_namespace }}"
+    from_file:
+      node-config.yaml: "{{ mktempout.stdout }}/node-config-{{ item.type }}.yaml"
+  with_items: "{{ openshift_master_node_configs }}"
+  run_once: true
+
+- name: remove templated files
+  file:
+    dest: "{{ mktempout.stdout }}/"
+    state: absent
+  with_items: "{{ openshift_master_node_configs }}"
+  run_once: true
diff --git a/roles/openshift_node/defaults/main.yml b/roles/openshift_node/defaults/main.yml
index 739b0d968..b310a8f64 100644
--- a/roles/openshift_node/defaults/main.yml
+++ b/roles/openshift_node/defaults/main.yml
@@ -4,7 +4,8 @@ openshift_node_debug_level: "{{ debug_level | default(2) }}"
 r_openshift_node_firewall_enabled: "{{ os_firewall_enabled | default(True) }}"
 r_openshift_node_use_firewalld: "{{ os_firewall_use_firewalld | default(False) }}"
 
-openshift_service_type: "{{ openshift.common.service_type }}"
+openshift_deployment_type: "{{ openshift_deployment_type | default('origin') }}"
+openshift_service_type: "{{ 'origin' if openshift_deployment_type == 'origin' else 'atomic-openshift' }}"
 
 openshift_image_tag: ''
 
@@ -17,7 +18,6 @@ openshift_node_ami_prep_packages:
 - openvswitch
 - docker
 - etcd
-#- pcs
 - haproxy
 - dnsmasq
 - ntp
@@ -54,7 +54,6 @@ openshift_node_ami_prep_packages:
 # - container-selinux
 # - atomic
 #
-openshift_deployment_type: origin
 
 openshift_node_bootstrap: False
 
diff --git a/roles/openshift_node/files/bootstrap.yml b/roles/openshift_node/files/bootstrap.yml
new file mode 100644
index 000000000..ea280640f
--- /dev/null
+++ b/roles/openshift_node/files/bootstrap.yml
@@ -0,0 +1,63 @@
+#!/usr/bin/ansible-playbook
+---
+- hosts: localhost
+  gather_facts: yes
+  vars:
+    origin_dns:
+      file: /etc/dnsmasq.d/origin-dns.conf
+      lines:
+      - regex: ^listen-address
+        state: present
+        line: "listen-address={{ ansible_default_ipv4.address }}"
+    node_dns:
+      file: /etc/dnsmasq.d/node-dnsmasq.conf
+      lines:
+      - regex: "^server=/in-addr.arpa/127.0.0.1$"
+        line: server=/in-addr.arpa/127.0.0.1
+      - regex: "^server=/cluster.local/127.0.0.1$"
+        line: server=/cluster.local/127.0.0.1
+
+  tasks:
+  - include_vars: openshift_settings.yaml
+
+  - name: set the data for node_dns
+    lineinfile:
+      create: yes
+      insertafter: EOF
+      path: "{{ node_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit) }}"
+    with_items: "{{ node_dns.lines }}"
+
+  - name: set the data for origin_dns
+    lineinfile:
+      create: yes
+      state: "{{ item.state | default('present') }}"
+      insertafter: "{{ item.after | default(omit) }}"
+      path: "{{ origin_dns.file }}"
+      regexp: "{{ item.regex }}"
+      line: "{{ item.line | default(omit)}}"
+    with_items: "{{ origin_dns.lines }}"
+
+  - when:
+    - openshift_group_type is defined
+    - openshift_group_type != ''
+    - openshift_group_type != 'master'
+    block:
+    - name: determine the openshift_service_type
+      stat:
+        path: /etc/sysconfig/atomic-openshift-node
+      register: service_type_results
+
+    - name: set openshift_service_type fact based on stat results
+      set_fact:
+        openshift_service_type: "{{ service_type_results.stat.exists | ternary('atomic-openshift', 'origin') }}"
+
+    - name: update the sysconfig to have necessary variables
+      lineinfile:
+        dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
+        line: "{{ item.line }}"
+        regexp: "{{ item.regexp }}"
+      with_items:
+      - line: "BOOTSTRAP_CONFIG_NAME=node-config-{{ openshift_group_type }}"
+        regexp: "^BOOTSTRAP_CONFIG_NAME=.*"
diff --git a/roles/openshift_node/handlers/main.yml b/roles/openshift_node/handlers/main.yml
index 25a6fc721..b102c1b18 100644
--- a/roles/openshift_node/handlers/main.yml
+++ b/roles/openshift_node/handlers/main.yml
@@ -3,7 +3,11 @@
   systemd:
     name: openvswitch
     state: restarted
-  when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift_node_use_openshift_sdn | bool
+  when:
+  - (not skip_node_svc_handlers | default(False) | bool)
+  - not (ovs_service_status_changed | default(false) | bool)
+  - openshift_node_use_openshift_sdn | bool
+  - not openshift_node_bootstrap
   register: l_openshift_node_stop_openvswitch_result
   until: not l_openshift_node_stop_openvswitch_result | failed
   retries: 3
@@ -11,10 +15,11 @@
   notify:
   - restart openvswitch pause
 
-
 - name: restart openvswitch pause
   pause: seconds=15
-  when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool
+  when:
+  - (not skip_node_svc_handlers | default(False) | bool)
+  - openshift.common.is_containerized | bool
 
 - name: restart node
   systemd:
diff --git a/roles/openshift_node/tasks/aws.yml b/roles/openshift_node/tasks/aws.yml
new file mode 100644
index 000000000..38c2b794d
--- /dev/null
+++ b/roles/openshift_node/tasks/aws.yml
@@ -0,0 +1,21 @@
+---
+- name: Configure AWS Cloud Provider Settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+    create: true
+  with_items:
+    - regex: '^AWS_ACCESS_KEY_ID='
+      line: "AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key | default('') }}"
+    - regex: '^AWS_SECRET_ACCESS_KEY='
+      line: "AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key | default('') }}"
+  register: sys_env_update
+  no_log: True
+  when:
+    - openshift_cloudprovider_kind is defined
+    - openshift_cloudprovider_kind == 'aws'
+    - openshift_cloudprovider_aws_access_key is defined
+    - openshift_cloudprovider_aws_secret_key is defined
+  notify:
+    - restart node
diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml
index 6bd2df362..8c03f6c41 100644
--- a/roles/openshift_node/tasks/bootstrap.yml
+++ b/roles/openshift_node/tasks/bootstrap.yml
@@ -17,17 +17,29 @@
       [Unit]
       After=cloud-init.service
 
-- name: update the sysconfig to have KUBECONFIG
+- name: update the sysconfig to have necessary variables
   lineinfile:
     dest: "/etc/sysconfig/{{ openshift_service_type }}-node"
-    line: "KUBECONFIG=/root/csr_kubeconfig"
+    line: "{{ item.line | default(omit) }}"
+    regexp: "{{ item.regexp }}"
+    state: "{{ item.state | default('present') }}"
+  with_items:
+  # add the kubeconfig
+  - line: "KUBECONFIG=/etc/origin/node/csr_kubeconfig"
     regexp: "^KUBECONFIG=.*"
+  # remove the config file.  This comes from openshift_facts
+  - regexp: "^CONFIG_FILE=.*"
+    state: absent
 
-- name: update the ExecStart to have bootstrap
-  lineinfile:
-    dest: "/usr/lib/systemd/system/{{ openshift_service_type }}-node.service"
-    line: "{% raw %}ExecStart=/usr/bin/openshift start node --bootstrap --kubeconfig=${KUBECONFIG} $OPTIONS{% endraw %}"
-    regexp: "^ExecStart=.*"
+- name: include aws sysconfig credentials
+  include: aws.yml
+  static: yes
+
+#- name: update the ExecStart to have bootstrap
+#  lineinfile:
+#    dest: "/usr/lib/systemd/system/{{ openshift_service_type }}-node.service"
+#    line: "{% raw %}ExecStart=/usr/bin/openshift start node --bootstrap --kubeconfig=${KUBECONFIG} $OPTIONS{% endraw %}"
+#    regexp: "^ExecStart=.*"
 
 - name: "disable {{ openshift_service_type }}-node and {{ openshift_service_type }}-master services"
   systemd:
@@ -42,6 +54,30 @@
     path: /etc/origin/.config_managed
   register: rpmgenerated_config
 
+- name: create directories for bootstrapping
+  file:
+    state: directory
+    dest: "{{ item }}"
+  with_items:
+  - /root/openshift_bootstrap
+  - /var/lib/origin/openshift.local.config
+  - /var/lib/origin/openshift.local.config/node
+  - "/etc/docker/certs.d/docker-registry.default.svc:5000"
+
+- name: laydown the bootstrap.yml file for on boot configuration
+  copy:
+    src: bootstrap.yml
+    dest: /root/openshift_bootstrap/bootstrap.yml
+
+- name: symlink master ca for docker-registry
+  file:
+    src: "{{ item }}"
+    dest: "/etc/docker/certs.d/docker-registry.default.svc:5000/{{ item | basename }}"
+    state: link
+    force: yes
+  with_items:
+  - /var/lib/origin/openshift.local.config/node/node-client-ca.crt
+
 - when: rpmgenerated_config.stat.exists
   block:
   - name: Remove RPM generated config files if present
@@ -50,6 +86,7 @@
       state: absent
     with_items:
     - master
+    - .config_managed
 
   # with_fileglob doesn't work correctly due to a few issues.
   # Could change this to fileglob when it gets fixed.
@@ -62,5 +99,7 @@
     file:
       path: "{{ item.path }}"
       state: absent
-    when: "'resolv.conf' not in item.path or 'node-dnsmasq.conf' not in item.path"
+    when:
+    - "'resolv.conf' not in item.path"
+    - "'node-dnsmasq.conf' not in item.path"
     with_items: "{{ find_results.files }}"
diff --git a/roles/openshift_node/tasks/config.yml b/roles/openshift_node/tasks/config.yml
index e5fcaf9af..c08f43118 100644
--- a/roles/openshift_node/tasks/config.yml
+++ b/roles/openshift_node/tasks/config.yml
@@ -46,26 +46,16 @@
   notify:
     - restart node
 
-- name: Configure AWS Cloud Provider Settings
-  lineinfile:
-    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
-    regexp: "{{ item.regex }}"
-    line: "{{ item.line }}"
-    create: true
-  with_items:
-    - regex: '^AWS_ACCESS_KEY_ID='
-      line: "AWS_ACCESS_KEY_ID={{ openshift_cloudprovider_aws_access_key | default('') }}"
-    - regex: '^AWS_SECRET_ACCESS_KEY='
-      line: "AWS_SECRET_ACCESS_KEY={{ openshift_cloudprovider_aws_secret_key | default('') }}"
-  no_log: True
-  when: openshift_cloudprovider_kind is defined and openshift_cloudprovider_kind == 'aws' and openshift_cloudprovider_aws_access_key is defined and openshift_cloudprovider_aws_secret_key is defined
-  notify:
-    - restart node
+- name: include aws provider credentials
+  include: aws.yml
+  static: yes
 
 # Necessary because when you're on a node that's also a master the master will be
 # restarted after the node restarts docker and it will take up to 60 seconds for
 # systemd to start the master again
-- when: openshift.common.is_containerized | bool
+- when:
+    - openshift.common.is_containerized | bool
+    - not openshift_node_bootstrap
   block:
     - name: Wait for master API to become available before proceeding
       # Using curl here since the uri module requires python-httplib2 and
@@ -90,26 +80,28 @@
         enabled: yes
         state: started
 
-- name: Start and enable node
-  systemd:
-    name: "{{ openshift.common.service_type }}-node"
-    enabled: yes
-    state: started
-    daemon_reload: yes
-  register: node_start_result
-  until: not node_start_result | failed
-  retries: 1
-  delay: 30
-  ignore_errors: true
+- when: not openshift_node_bootstrap
+  block:
+    - name: Start and enable node
+      systemd:
+        name: "{{ openshift.common.service_type }}-node"
+        enabled: yes
+        state: started
+        daemon_reload: yes
+      register: node_start_result
+      until: not node_start_result | failed
+      retries: 1
+      delay: 30
+      ignore_errors: true
 
-- name: Dump logs from node service if it failed
-  command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
-  when: node_start_result | failed
+    - name: Dump logs from node service if it failed
+      command: journalctl --no-pager -n 100 -u {{ openshift.common.service_type }}-node
+      when: node_start_result | failed
 
-- name: Abort if node failed to start
-  fail:
-    msg: Node failed to start please inspect the logs and try again
-  when: node_start_result | failed
+    - name: Abort if node failed to start
+      fail:
+        msg: Node failed to start please inspect the logs and try again
+      when: node_start_result | failed
 
-- set_fact:
-    node_service_status_changed: "{{ node_start_result | changed }}"
+    - set_fact:
+        node_service_status_changed: "{{ node_start_result | changed }}"
diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml
index 1539d6e3b..6b7e40491 100644
--- a/roles/openshift_node/tasks/install.yml
+++ b/roles/openshift_node/tasks/install.yml
@@ -3,12 +3,12 @@
   block:
   - name: Install Node package
     package:
-      name: "{{ openshift.common.service_type }}-node{{ openshift_pkg_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
 
   - name: Install sdn-ovs package
     package:
-      name: "{{ openshift.common.service_type }}-sdn-ovs{{ openshift_pkg_version | oo_image_tag_to_rpm_version(include_dash=True) }}"
+      name: "{{ openshift.common.service_type }}-sdn-ovs{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
     when:
     - openshift_node_use_openshift_sdn | bool
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index 59b8bb76e..eae9ca7bc 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -66,15 +66,10 @@
     sysctl_file: "/etc/sysctl.d/99-openshift.conf"
     reload: yes
 
-- name: include bootstrap node config
-  include: bootstrap.yml
-  when: openshift_node_bootstrap
-
 - include: registry_auth.yml
 
 - name: include standard node config
   include: config.yml
-  when: not openshift_node_bootstrap
 
 #### Storage class plugins here ####
 - name: NFS storage plugin configuration
@@ -98,3 +93,7 @@
 
 - include: config/workaround-bz1331590-ovs-oom-fix.yml
   when: openshift_node_use_openshift_sdn | default(true) | bool
+
+- name: include bootstrap node config
+  include: bootstrap.yml
+  when: openshift_node_bootstrap
diff --git a/roles/openshift_node/templates/node.service.j2 b/roles/openshift_node/templates/node.service.j2
index 0856737f6..7602d8ee6 100644
--- a/roles/openshift_node/templates/node.service.j2
+++ b/roles/openshift_node/templates/node.service.j2
@@ -12,17 +12,17 @@ After=dnsmasq.service
 
 [Service]
 Type=notify
-EnvironmentFile=/etc/sysconfig/{{ openshift.common.service_type }}-node
+EnvironmentFile=/etc/sysconfig/{{ openshift_service_type }}-node
 Environment=GOTRACEBACK=crash
 ExecStartPre=/usr/bin/cp /etc/origin/node/node-dnsmasq.conf /etc/dnsmasq.d/
 ExecStartPre=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:/in-addr.arpa/127.0.0.1,/{{ openshift.common.dns_domain }}/127.0.0.1
 ExecStopPost=/usr/bin/rm /etc/dnsmasq.d/node-dnsmasq.conf
 ExecStopPost=/usr/bin/dbus-send --system --dest=uk.org.thekelleys.dnsmasq /uk/org/thekelleys/dnsmasq uk.org.thekelleys.SetDomainServers array:string:
-ExecStart=/usr/bin/openshift start node --config=${CONFIG_FILE} $OPTIONS
+ExecStart=/usr/bin/openshift start node {% if openshift_node_bootstrap %} --kubeconfig=${KUBECONFIG} --bootstrap-config-name=${BOOTSTRAP_CONFIG_NAME}{% endif %} --config=${CONFIG_FILE} $OPTIONS
 LimitNOFILE=65536
 LimitCORE=infinity
 WorkingDirectory=/var/lib/origin/
-SyslogIdentifier={{ openshift.common.service_type }}-node
+SyslogIdentifier={{ openshift_service_type }}-node
 Restart=always
 RestartSec=5s
 TimeoutStartSec=300