Merge pull request #763 from openshift/master

Merge master into prod.

18 files changed, 518 insertions, 46 deletions

diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml
index 14ffa928f..4c74f96db 100644
--- a/playbooks/common/openshift-cluster/config.yml
+++ b/playbooks/common/openshift-cluster/config.yml
@@ -1,4 +1,74 @@
 ---
+- name: Populate config host groups
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - fail:
+      msg: This playbook rquires g_etcd_group to be set
+    when: g_etcd_group is not defined
+
+  - fail:
+      msg: This playbook rquires g_masters_group to be set
+    when: g_masters_group is not defined
+
+  - fail:
+      msg: This playbook rquires g_nodes_group to be set
+    when: g_nodes_group is not defined
+
+  - name: Evaluate oo_etcd_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_etcd_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_etcd_group] | default([])
+
+  - name: Evaluate oo_masters_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_masters_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_masters_group] | default([])
+
+  - name: Evaluate oo_nodes_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nodes_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_nodes_group] | default([])
+
+  - name: Evaluate oo_nodes_to_config
+    add_host:
+      name: "{{ item }}"
+      groups: oo_nodes_to_config
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    with_items: groups[g_masters_group] | default([])
+    when: g_nodeonmaster is defined and g_nodeonmaster == true
+
+  - name: Evaluate oo_first_etcd
+    add_host:
+      name: "{{ groups[g_etcd_group][0] }}"
+      groups: oo_first_etcd
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    when: g_etcd_group in groups and (groups[g_etcd_group] | length) > 0
+
+  - name: Evaluate oo_first_master
+    add_host:
+      name: "{{ groups[g_masters_group][0] }}"
+      groups: oo_first_master
+      ansible_ssh_user: "{{ g_ssh_user | default(omit) }}"
+      ansible_sudo: "{{ g_sudo | default(omit) }}"
+    when: g_masters_group in groups and (groups[g_masters_group] | length) > 0
+
+- include: ../openshift-etcd/config.yml
+
 - include: ../openshift-master/config.yml
 
 - include: ../openshift-node/config.yml
+  vars:
+    osn_cluster_dns_domain: "{{ hostvars[groups.oo_first_master.0].openshift.dns.domain }}"
+    osn_cluster_dns_ip: "{{ hostvars[groups.oo_first_master.0].openshift.dns.ip }}"
diff --git a/playbooks/common/openshift-cluster/create_services.yml b/playbooks/common/openshift-cluster/create_services.yml
deleted file mode 100644
index e70709d19..000000000
--- a/playbooks/common/openshift-cluster/create_services.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-- name: Deploy OpenShift Services
-  hosts: "{{ g_svc_master }}"
-  connection: ssh
-  gather_facts: yes
-  roles:
-  - openshift_registry
-  - openshift_router
diff --git a/playbooks/common/openshift-cluster/lookup_plugins b/playbooks/common/openshift-cluster/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-cluster/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-cluster/set_etcd_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/set_etcd_launch_facts_tasks.yml
new file mode 100644
index 000000000..1a6580795
--- /dev/null
+++ b/playbooks/common/openshift-cluster/set_etcd_launch_facts_tasks.yml
@@ -0,0 +1,13 @@
+---
+- set_fact: k8s_type="etcd"
+
+- name: Generate etcd instance names(s)
+  set_fact:
+    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
+  register: etcd_names_output
+  with_sequence: count={{ num_etcd }}
+
+- set_fact:
+    etcd_names: "{{ etcd_names_output.results | default([])
+                    | oo_collect('ansible_facts')
+                    | oo_collect('scratch_name') }}"
diff --git a/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml
index 118727273..36d7b7870 100644
--- a/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml
+++ b/playbooks/common/openshift-cluster/set_master_launch_facts_tasks.yml
@@ -5,7 +5,9 @@
   set_fact:
     scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
   register: master_names_output
-  with_sequence: start=1 end={{ num_masters }}
+  with_sequence: count={{ num_masters }}
 
 - set_fact:
-    master_names: "{{ master_names_output.results | oo_collect('ansible_facts') | oo_collect('scratch_name') }}"
+    master_names: "{{ master_names_output.results | default([])
+                      | oo_collect('ansible_facts')
+                      | oo_collect('scratch_name') }}"
diff --git a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml b/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
index 162315d46..278942f8b 100644
--- a/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
+++ b/playbooks/common/openshift-cluster/set_node_launch_facts_tasks.yml
@@ -1,11 +1,15 @@
 ---
-- set_fact: k8s_type="node"
+- set_fact: k8s_type=node
+- set_fact: sub_host_type="{{ type }}"
+- set_fact: number_nodes="{{ count }}"
 
 - name: Generate node instance names(s)
   set_fact:
-    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ '%05x' | format(1048576 | random) }}"
+    scratch_name: "{{ cluster_id }}-{{ k8s_type }}-{{ sub_host_type }}-{{ '%05x' | format(1048576 | random) }}"
   register: node_names_output
-  with_sequence: start=1 end={{ num_nodes }}
+  with_sequence: count={{ number_nodes }}
 
 - set_fact:
-    node_names: "{{ node_names_output.results | oo_collect('ansible_facts') | oo_collect('scratch_name') }}"
+    node_names: "{{ node_names_output.results | default([])
+                    | oo_collect('ansible_facts')
+                    | oo_collect('scratch_name') }}"
diff --git a/playbooks/common/openshift-cluster/update_repos_and_packages.yml b/playbooks/common/openshift-cluster/update_repos_and_packages.yml
index e92c6f1ee..190e2d862 100644
--- a/playbooks/common/openshift-cluster/update_repos_and_packages.yml
+++ b/playbooks/common/openshift-cluster/update_repos_and_packages.yml
@@ -3,5 +3,10 @@
   vars:
     openshift_deployment_type: "{{ deployment_type }}"
   roles:
+  - role: rhel_subscribe
+    when: deployment_type == "enterprise" and
+          ansible_distribution == "RedHat" and
+          lookup('oo_option', 'rhel_skip_subscription') | default(rhsub_skip, True) |
+            default('no', True) | lower in ['no', 'false']
   - openshift_repos
   - os_update_latest
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
new file mode 100644
index 000000000..952960652
--- /dev/null
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -0,0 +1,97 @@
+---
+- name: Set etcd facts needed for generating certs
+  hosts: oo_etcd_to_config
+  roles:
+  - openshift_facts
+  tasks:
+  - openshift_facts:
+      role: "{{ item.role }}"
+      local_facts: "{{ item.local_facts }}"
+    with_items:
+      - role: common
+        local_facts:
+          hostname: "{{ openshift_hostname | default(None) }}"
+          public_hostname: "{{ openshift_public_hostname | default(None) }}"
+          deployment_type: "{{ openshift_deployment_type }}"
+  - name: Check status of etcd certificates
+    stat:
+      path: "{{ item }}"
+    with_items:
+    - /etc/etcd/server.crt
+    - /etc/etcd/peer.crt
+    - /etc/etcd/ca.crt
+    register: g_etcd_server_cert_stat_result
+  - set_fact:
+      etcd_server_certs_missing: "{{ g_etcd_server_cert_stat_result.results | map(attribute='stat.exists')
+                                    | list | intersect([false])}}"
+      etcd_cert_subdir: etcd-{{ openshift.common.hostname }}
+      etcd_cert_config_dir: /etc/etcd
+      etcd_cert_prefix:
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_etcd_mktemp
+    changed_when: False
+
+- name: Configure etcd certificates
+  hosts: oo_first_etcd
+  vars:
+    etcd_generated_certs_dir: /etc/etcd/generated_certs
+    etcd_needing_server_certs: "{{ hostvars
+                                  | oo_select_keys(groups['oo_etcd_to_config'])
+                                  | oo_filter_list(filter_attr='etcd_server_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}"
+  roles:
+  - etcd_certificates
+  post_tasks:
+  - name: Create a tarball of the etcd certs
+    command: >
+      tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz
+        -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} .
+    args:
+      creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+    with_items: etcd_needing_server_certs
+  - name: Retrieve the etcd cert tarballs
+    fetch:
+      src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_needing_server_certs
+
+- name: Configure etcd hosts
+  hosts: oo_etcd_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_etcd_mktemp.stdout }}"
+    etcd_url_scheme: https
+    etcd_peer_url_scheme: https
+    etcd_peers_group: oo_etcd_to_config
+  pre_tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ etcd_cert_config_dir }}"
+      state: directory
+  - name: Unarchive the tarball on the etcd host
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz"
+      dest: "{{ etcd_cert_config_dir }}"
+    when: etcd_server_certs_missing
+  roles:
+  - etcd
+  - role: nickhammond.logrotate
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - file: name={{ g_etcd_mktemp.stdout }} state=absent
+    changed_when: False
diff --git a/playbooks/common/openshift-etcd/filter_plugins b/playbooks/common/openshift-etcd/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/common/openshift-etcd/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-etcd/lookup_plugins b/playbooks/common/openshift-etcd/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-etcd/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-etcd/roles b/playbooks/common/openshift-etcd/roles
new file mode 120000
index 000000000..e2b799b9d
--- /dev/null
+++ b/playbooks/common/openshift-etcd/roles
@@ -0,0 +1 @@
+../../../roles/
\ No newline at end of file
diff --git a/playbooks/common/openshift-etcd/service.yml b/playbooks/common/openshift-etcd/service.yml
new file mode 100644
index 000000000..0bf69b22f
--- /dev/null
+++ b/playbooks/common/openshift-etcd/service.yml
@@ -0,0 +1,18 @@
+---
+- name: Populate g_service_masters host group if needed
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - fail: msg="new_cluster_state is required to be injected in this playbook"
+    when: new_cluster_state is not defined
+
+  - name: Evaluate g_service_etcd
+    add_host: name={{ item }} groups=g_service_etcd
+    with_items: oo_host_group_exp | default([])
+
+- name: Change etcd state on etcd instance(s)
+  hosts: g_service_etcd
+  connection: ssh
+  gather_facts: no
+  tasks:
+    - service: name=etcd state="{{ new_cluster_state }}"
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 052ed14c7..1dec923fc 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -1,18 +1,280 @@
 ---
+- name: Set master facts and determine if external etcd certs need to be generated
+  hosts: oo_masters_to_config
+  pre_tasks:
+  - name: Check for RPM generated config marker file .config_managed
+    stat:
+      path: /etc/origin/.config_managed
+    register: rpmgenerated_config
+
+  - name: Remove RPM generated config files if present
+    file:
+      path: "/etc/origin/{{ item }}"
+      state: absent
+    when: rpmgenerated_config.stat.exists == true and deployment_type in ['openshift-enterprise', 'atomic-enterprise']
+    with_items:
+    - master
+    - node
+    - .config_managed
+
+  - set_fact:
+      openshift_master_etcd_port: "{{ (etcd_client_port | default('2379')) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else none }}"
+      openshift_master_etcd_hosts: "{{ hostvars
+                                       | oo_select_keys(groups['oo_etcd_to_config']
+                                                        | default([]))
+                                       | oo_collect('openshift.common.hostname')
+                                       | default(none, true) }}"
+  roles:
+  - openshift_facts
+  post_tasks:
+  - openshift_facts:
+      role: "{{ item.role }}"
+      local_facts: "{{ item.local_facts }}"
+    with_items:
+      - role: common
+        local_facts:
+          hostname: "{{ openshift_hostname | default(None) }}"
+          public_hostname: "{{ openshift_public_hostname | default(None) }}"
+          deployment_type: "{{ openshift_deployment_type }}"
+      - role: master
+        local_facts:
+          api_port: "{{ openshift_master_api_port | default(None) }}"
+          api_url: "{{ openshift_master_api_url | default(None) }}"
+          api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
+          public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
+          cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+          cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+          cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
+          console_path: "{{ openshift_master_console_path | default(None) }}"
+          console_port: "{{ openshift_master_console_port | default(None) }}"
+          console_url: "{{ openshift_master_console_url | default(None) }}"
+          console_use_ssl: "{{ openshift_master_console_use_ssl | default(None) }}"
+          public_console_url: "{{ openshift_master_public_console_url | default(None) }}"
+  - name: Check status of external etcd certificatees
+    stat:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+    with_items:
+    - master.etcd-client.crt
+    - master.etcd-ca.crt
+    register: g_external_etcd_cert_stat_result
+  - set_fact:
+      etcd_client_certs_missing: "{{ g_external_etcd_cert_stat_result.results
+                                    | map(attribute='stat.exists')
+                                    | list | intersect([false])}}"
+      etcd_cert_subdir: openshift-master-{{ openshift.common.hostname }}
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: master.etcd-
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+
+- name: Create temp directory for syncing certs
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: g_master_mktemp
+    changed_when: False
+
+- name: Configure etcd certificates
+  hosts: oo_first_etcd
+  vars:
+    etcd_generated_certs_dir: /etc/etcd/generated_certs
+    etcd_needing_client_certs: "{{ hostvars
+                                   | oo_select_keys(groups['oo_masters_to_config'])
+                                   | oo_filter_list(filter_attr='etcd_client_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  roles:
+  - etcd_certificates
+  post_tasks:
+  - name: Create a tarball of the etcd certs
+    command: >
+      tar -czvf {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz
+        -C {{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }} .
+    args:
+      creates: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+    with_items: etcd_needing_client_certs
+  - name: Retrieve the etcd cert tarballs
+    fetch:
+      src: "{{ etcd_generated_certs_dir }}/{{ item.etcd_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: etcd_needing_client_certs
+
+- name: Copy the external etcd certs to the masters
+  hosts: oo_masters_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ openshift.common.config_base }}/master"
+      state: directory
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ etcd_cert_subdir }}.tgz"
+      dest: "{{ etcd_cert_config_dir }}"
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+  - file:
+      path: "{{ etcd_cert_config_dir }}/{{ item }}"
+      owner: root
+      group: root
+      mode: 0600
+    with_items:
+    - master.etcd-client.crt
+    - master.etcd-client.key
+    - master.etcd-ca.crt
+    when: etcd_client_certs_missing is defined and etcd_client_certs_missing
+
+- name: Determine if master certificates need to be generated
+  hosts: oo_masters_to_config
+  tasks:
+  - set_fact:
+      openshift_master_certs_no_etcd:
+      - admin.crt
+      - master.kubelet-client.crt
+      - "{{ 'master.proxy-client.crt' if openshift.common.version_greater_than_3_1_or_1_1 else omit }}"
+      - master.server.crt
+      - openshift-master.crt
+      - openshift-registry.crt
+      - openshift-router.crt
+      - etcd.server.crt
+      openshift_master_certs_etcd:
+      - master.etcd-client.crt
+
+  - set_fact:
+      openshift_master_certs: "{{ (openshift_master_certs_no_etcd | union(openshift_master_certs_etcd)) if (groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config) else openshift_master_certs_no_etcd }}"
+
+  - name: Check status of master certificates
+    stat:
+      path: "{{ openshift.common.config_base }}/master/{{ item }}"
+    with_items: openshift_master_certs
+    register: g_master_cert_stat_result
+  - set_fact:
+      master_certs_missing: "{{ False in (g_master_cert_stat_result.results
+                                | map(attribute='stat.exists')
+                                | list ) }}"
+      master_cert_subdir: master-{{ openshift.common.hostname }}
+      master_cert_config_dir: "{{ openshift.common.config_base }}/master"
+
+- name: Configure master certificates
+  hosts: oo_first_master
+  vars:
+    master_generated_certs_dir: "{{ openshift.common.config_base }}/generated-configs"
+    masters_needing_certs: "{{ hostvars
+                               | oo_select_keys(groups['oo_masters_to_config'] | difference(groups['oo_first_master']))
+                               | oo_filter_list(filter_attr='master_certs_missing') }}"
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+  roles:
+  - openshift_master_certificates
+  post_tasks:
+  - name: Remove generated etcd client certs when using external etcd
+    file:
+      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+      state: absent
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    with_nested:
+    - masters_needing_certs
+    - - master.etcd-client.crt
+      - master.etcd-client.key
+
+  - name: Create a tarball of the master certs
+    command: >
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
+    args:
+      creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
+    with_items: masters_needing_certs
+
+  - name: Retrieve the master cert tarball from the master
+    fetch:
+      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
+      dest: "{{ sync_tmpdir }}/"
+      flat: yes
+      fail_on_missing: yes
+      validate_checksum: yes
+    with_items: masters_needing_certs
+
 - name: Configure master instances
   hosts: oo_masters_to_config
+  vars:
+    sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    embedded_etcd: "{{ openshift.master.embedded_etcd }}"
+  pre_tasks:
+  - name: Ensure certificate directory exists
+    file:
+      path: "{{ openshift.common.config_base }}/master"
+      state: directory
+    when: master_certs_missing and 'oo_first_master' not in group_names
+  - name: Unarchive the tarball on the master
+    unarchive:
+      src: "{{ sync_tmpdir }}/{{ master_cert_subdir }}.tgz"
+      dest: "{{ master_cert_config_dir }}"
+    when: master_certs_missing and 'oo_first_master' not in group_names
   roles:
   - openshift_master
+  - role: nickhammond.logrotate
   - role: fluentd_master
     when: openshift.common.use_fluentd | bool
-  tasks:
+  post_tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
+- name: Additional master configuration
+  hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
+  roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool
+  - openshift_examples
+  - role: openshift_cluster_metrics
+    when: openshift.common.use_cluster_metrics | bool
+
+- name: Enable cockpit
+  hosts: oo_first_master
+  vars:
+    cockpit_plugins: "{{ osm_cockpit_plugins | default(['cockpit-kubernetes']) }}"
+  roles:
+  - role: cockpit
+    when: ( deployment_type in ['atomic-enterprise','openshift-enterprise'] ) and
+      (osm_use_cockpit | bool or osm_use_cockpit is undefined )
+
 # Additional instance config for online deployments
 - name: Additional instance config
   hosts: oo_masters_deployment_type_online
   roles:
   - pods
   - os_env_extras
+
+- name: Delete temporary directory on localhost
+  hosts: localhost
+  connection: local
+  sudo: false
+  gather_facts: no
+  tasks:
+  - file: name={{ g_master_mktemp.stdout }} state=absent
+    changed_when: False
+
+- name: Configure service accounts
+  hosts: oo_first_master
+
+  vars:
+    accounts: ["router", "registry"]
+
+  roles:
+  - openshift_serviceaccounts
+
+- name: Create services
+  hosts: oo_first_master
+  roles:
+  - role: openshift_router
+    when: openshift.master.infra_nodes is defined
+  #- role: openshift_registry
diff --git a/playbooks/common/openshift-master/lookup_plugins b/playbooks/common/openshift-master/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-master/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-master/service.yml b/playbooks/common/openshift-master/service.yml
index 5636ad156..27e1e66f9 100644
--- a/playbooks/common/openshift-master/service.yml
+++ b/playbooks/common/openshift-master/service.yml
@@ -10,9 +10,9 @@
     add_host: name={{ item }} groups=g_service_masters
     with_items: oo_host_group_exp | default([])
 
-- name: Change openshift-master state on master instance(s)
+- name: Change state on master instance(s)
   hosts: g_service_masters
   connection: ssh
   gather_facts: no
   tasks:
-    - service: name=openshift-master state="{{ new_cluster_state }}"
+    - service: name={{ openshift.common.service_type }}-master state="{{ new_cluster_state }}"
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 9e642f3d3..a14ca8e11 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -18,26 +18,26 @@
           deployment_type: "{{ openshift_deployment_type }}"
       - role: node
         local_facts:
-          resources_cpu: "{{ openshift_node_resources_cpu | default(None) }}"
-          resources_memory: "{{ openshift_node_resources_memory | default(None) }}"
-          pod_cidr: "{{ openshift_node_pod_cidr | default(None) }}"
           labels: "{{ openshift_node_labels | default(None) }}"
           annotations: "{{ openshift_node_annotations | default(None) }}"
+          schedulable: "{{ openshift_schedulable | default(openshift_scheduleable) | default(None) }}"
   - name: Check status of node certificates
     stat:
-      path: "{{ item }}"
+      path: "{{ openshift.common.config_base }}/node/{{ item }}"
     with_items:
-    - "/etc/openshift/node/node.key"
-    - "/etc/openshift/node/node.kubeconfig"
-    - "/etc/openshift/node/ca.crt"
-    - "/etc/openshift/node/server.key"
+    - "system:node:{{ openshift.common.hostname }}.crt"
+    - "system:node:{{ openshift.common.hostname }}.key"
+    - "system:node:{{ openshift.common.hostname }}.kubeconfig"
+    - ca.crt
+    - server.key
+    - server.crt
     register: stat_result
   - set_fact:
       certs_missing: "{{ stat_result.results | map(attribute='stat.exists')
                          | list | intersect([false])}}"
       node_subdir: node-{{ openshift.common.hostname }}
-      config_dir: /etc/openshift/generated-configs/node-{{ openshift.common.hostname }}
-      node_cert_dir: /etc/openshift/node
+      config_dir: "{{ openshift.common.config_base }}/generated-configs/node-{{ openshift.common.hostname }}"
+      node_cert_dir: "{{ openshift.common.config_base }}/node"
 
 - name: Create temp directory for syncing certs
   hosts: localhost
@@ -50,17 +50,16 @@
     register: mktemp
     changed_when: False
 
-- name: Register nodes
+- name: Create node certificates
   hosts: oo_first_master
   vars:
     nodes_needing_certs: "{{ hostvars
-                             | oo_select_keys(groups['oo_nodes_to_config'])
+                             | oo_select_keys(groups['oo_nodes_to_config']
+                                              | default([]))
                              | oo_filter_list(filter_attr='certs_missing') }}"
-    openshift_nodes: "{{ hostvars
-                         | oo_select_keys(groups['oo_nodes_to_config']) }}"
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
   roles:
-  - openshift_register_nodes
+  - openshift_node_certificates
   post_tasks:
   - name: Create a tarball of the node config directories
     command: >
@@ -84,16 +83,16 @@
   hosts: oo_nodes_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
-    openshift_node_master_api_url: "{{ hostvars[openshift_first_master].openshift.master.api_url }}"
+    openshift_node_master_api_url: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_url }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
       path: "{{ node_cert_dir }}"
       state: directory
 
-  # TODO: notify restart openshift-node
+  # TODO: notify restart node
   # possibly test service started time against certificate/config file
-  # timestamps in openshift-node to trigger notify
+  # timestamps in node to trigger notify
   - name: Unarchive the tarball on the node
     unarchive:
       src: "{{ sync_tmpdir }}/{{ node_subdir }}.tgz"
@@ -101,6 +100,7 @@
     when: certs_missing
   roles:
   - openshift_node
+  - role: nickhammond.logrotate
   - role: fluentd_node
     when: openshift.common.use_fluentd | bool
   tasks:
@@ -108,15 +108,6 @@
     group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Delete the temporary directory on the master
-  hosts: oo_first_master
-  gather_facts: no
-  vars:
-    sync_tmpdir: "{{ hostvars.localhost.mktemp.stdout }}"
-  tasks:
-  - file: name={{ sync_tmpdir }} state=absent
-    changed_when: False
-
 - name: Delete temporary directory on localhost
   hosts: localhost
   connection: local
@@ -133,3 +124,15 @@
   roles:
   - os_env_extras
   - os_env_extras_node
+
+- name: Set schedulability
+  hosts: oo_first_master
+  vars:
+    openshift_nodes: "{{ hostvars
+                         | oo_select_keys(groups['oo_nodes_to_config'])
+                         | oo_collect('openshift.common.hostname') }}"
+    openshift_node_vars: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config']) }}"
+  pre_tasks:
+
+  roles:
+  - openshift_manage_node
diff --git a/playbooks/common/openshift-node/lookup_plugins b/playbooks/common/openshift-node/lookup_plugins
new file mode 120000
index 000000000..ac79701db
--- /dev/null
+++ b/playbooks/common/openshift-node/lookup_plugins
@@ -0,0 +1 @@
+../../../lookup_plugins
\ No newline at end of file
diff --git a/playbooks/common/openshift-node/service.yml b/playbooks/common/openshift-node/service.yml
index f76df089f..5cf83e186 100644
--- a/playbooks/common/openshift-node/service.yml
+++ b/playbooks/common/openshift-node/service.yml
@@ -10,9 +10,9 @@
     add_host: name={{ item }} groups=g_service_nodes
     with_items: oo_host_group_exp | default([])
 
-- name: Change openshift-node state on node instance(s)
+- name: Change state on node instance(s)
   hosts: g_service_nodes
   connection: ssh
   gather_facts: no
   tasks:
-    - service: name=openshift-node state="{{ new_cluster_state }}"
+    - service: name={{ service_type }}-node state="{{ new_cluster_state }}"