From 6b4282004a4331d9db0e0ab857c96d83a738d82c Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Tue, 14 Jul 2015 14:48:38 -0400 Subject: Initial HA master - Ability to specify multiple masters - configures the CA only a single time on the first master - creates and distributes additional certs for additional master hosts - Depending on the status of openshift_master_cluster_defer_ha (defaults to False) one of two actions are taken when multiple masters are defined 1. If openshift_master_cluster_defer_ha is true a. Certs/configs for all masters are deployed b. openshift-master service is only started and enabled on the master c. HA configuration is expected to be handled by the user manually after the completion of the playbook run. 2. If oepnshift_master_cluster_defer_ha is false or undefined a. Certs/configs for all masters are deployed b. a Pacemaker/RHEL HA cluster is configured i. VIPs are configured based on the values of openshift_master_cluster_vip and openshift_master_cluster_plublic_vip ii. The openshift-master service is configured as an active/passive cluster service --- inventory/byo/hosts.example | 20 +++++++++- playbooks/common/openshift-master/config.yml | 29 ++++++++++++--- roles/openshift_facts/library/openshift_facts.py | 42 ++++++++++++++++++--- roles/openshift_master/defaults/main.yml | 6 +++ roles/openshift_master/handlers/main.yml | 1 + roles/openshift_master/tasks/main.yml | 21 +++++++++++ roles/openshift_master_ca/tasks/main.yml | 2 +- roles/openshift_master_certificates/tasks/main.yml | 16 ++++++-- roles/openshift_master_certificates/vars/main.yml | 3 -- roles/openshift_master_cluster/README.md | 34 +++++++++++++++++ roles/openshift_master_cluster/meta/main.yml | 16 ++++++++ roles/openshift_master_cluster/tasks/configure.yml | 43 ++++++++++++++++++++++ .../tasks/configure_deferred.yml | 8 ++++ roles/openshift_master_cluster/tasks/main.yml | 13 +++++++ roles/openshift_node_certificates/tasks/main.yml | 2 +- 15 files changed, 234 insertions(+), 22 deletions(-) create mode 100644 roles/openshift_master_cluster/README.md create mode 100644 roles/openshift_master_cluster/meta/main.yml create mode 100644 roles/openshift_master_cluster/tasks/configure.yml create mode 100644 roles/openshift_master_cluster/tasks/configure_deferred.yml create mode 100644 roles/openshift_master_cluster/tasks/main.yml diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example index 41216dd89..56f4da5a2 100644 --- a/inventory/byo/hosts.example +++ b/inventory/byo/hosts.example @@ -38,14 +38,30 @@ deployment_type=enterprise # Allow all auth #openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}] +# master cluster ha variables using pacemaker or RHEL HA +#openshift_master_cluster_password=openshift_cluster +#openshift_master_cluster_vip=192.168.133.25 +#openshift_master_cluster_public_vip=192.168.133.25 +#openshift_master_cluster_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com + +# master cluster ha variables when using a different HA solution +# For installation the value of openshift_master_cluster_hostname must resolve +# to the first master defined in the inventory. +# The HA solution must be manually configured after installation and must ensure +# that openshift-master is running on a single master host. +#openshift_master_cluster_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com +#openshift_master_cluster_defer_ha=True + # host group for masters [masters] -ose3-master-ansible.test.example.com +ose3-master[1:3]-ansible.test.example.com [etcd] ose3-etcd[1:3]-ansible.test.example.com # host group for nodes [nodes] -ose3-master-ansible.test.example.com openshift_scheduleable=False +ose3-master[1:3]-ansible.test.example.com openshift_scheduleable=False ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 3956128e1..904ad2dab 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -27,6 +27,9 @@ api_url: "{{ openshift_master_api_url | default(None) }}" api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}" public_api_url: "{{ openshift_master_public_api_url | default(None) }}" + cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" + cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" + cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}" console_path: "{{ openshift_master_console_path | default(None) }}" console_port: "{{ openshift_master_console_port | default(None) }}" console_url: "{{ openshift_master_console_url | default(None) }}" @@ -152,16 +155,26 @@ roles: - openshift_master_certificates post_tasks: + - name: Remove generated etcd client certs when using external etcd + file: + path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: absent + when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config + with_nested: + - masters_needing_certs + - - master.etcd-client.crt + - master.etcd-client.key + - name: Create a tarball of the master certs command: > - tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz - -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} . + tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz + -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} . args: - creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" + creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" with_items: masters_needing_certs - name: Retrieve the master cert tarball from the master fetch: - src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz" + src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz" dest: "{{ sync_tmpdir }}/" flat: yes fail_on_missing: yes @@ -172,6 +185,7 @@ hosts: oo_masters_to_config vars: sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" pre_tasks: - name: Ensure certificate directory exists file: @@ -192,9 +206,14 @@ group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }} changed_when: False -- name: Deploy OpenShift examples +- name: Additional master configuration hosts: oo_first_master + vars: + openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}" + omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}" roles: + - role: openshift_master_cluster + when: openshift_master_ha | bool - openshift_examples # Additional instance config for online deployments diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 727861b07..d733639c3 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -367,9 +367,11 @@ def set_url_facts_if_unset(facts): console_path = facts['master']['console_path'] etcd_use_ssl = facts['master']['etcd_use_ssl'] etcd_hosts = facts['master']['etcd_hosts'] - etcd_port = facts['master']['etcd_port'], + etcd_port = facts['master']['etcd_port'] hostname = facts['common']['hostname'] public_hostname = facts['common']['public_hostname'] + cluster_hostname = facts['master'].get('cluster_hostname') + cluster_public_hostname = facts['master'].get('cluster_public_hostname') if 'etcd_urls' not in facts['master']: etcd_urls = [] @@ -384,24 +386,51 @@ def set_url_facts_if_unset(facts): etcd_port)] facts['master']['etcd_urls'] = etcd_urls if 'api_url' not in facts['master']: - facts['master']['api_url'] = format_url(api_use_ssl, hostname, + api_hostname = cluster_hostname if cluster_hostname else hostname + facts['master']['api_url'] = format_url(api_use_ssl, api_hostname, api_port) if 'public_api_url' not in facts['master']: + api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname facts['master']['public_api_url'] = format_url(api_use_ssl, - public_hostname, + api_public_hostname, api_port) if 'console_url' not in facts['master']: + console_hostname = cluster_hostname if cluster_hostname else hostname facts['master']['console_url'] = format_url(console_use_ssl, - hostname, + console_hostname, console_port, console_path) if 'public_console_url' not in facts['master']: + console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname facts['master']['public_console_url'] = format_url(console_use_ssl, - public_hostname, + console_public_hostname, console_port, console_path) return facts +def set_aggregate_facts(facts): + """ Set aggregate facts + + Args: + facts (dict): existing facts + Returns: + dict: the facts dict updated with aggregated facts + """ + all_hostnames = set() + if 'common' in facts: + all_hostnames.add(facts['common']['hostname']) + all_hostnames.add(facts['common']['public_hostname']) + + if 'master' in facts: + if 'cluster_hostname' in facts['master']: + all_hostnames.add(facts['master']['cluster_hostname']) + if 'cluster_public_hostname' in facts['master']: + all_hostnames.add(facts['master']['cluster_public_hostname']) + + facts['common']['all_hostnames'] = list(all_hostnames) + + return facts + def set_sdn_facts_if_unset(facts): """ Set sdn facts if not already present in facts dict @@ -675,6 +704,7 @@ class OpenShiftFacts(object): facts = set_identity_providers_if_unset(facts) facts = set_registry_url_if_unset(facts) facts = set_sdn_facts_if_unset(facts) + facts = set_aggregate_facts(facts) return dict(openshift=facts) def get_defaults(self, roles): @@ -713,7 +743,7 @@ class OpenShiftFacts(object): session_name='ssn', session_secrets_file='', access_token_max_seconds=86400, auth_token_max_seconds=500, - oauth_grant_method='auto') + oauth_grant_method='auto', cluster_defer_ha=False) defaults['master'] = master if 'node' in roles: diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml index 11195e83e..ca8860099 100644 --- a/roles/openshift_master/defaults/main.yml +++ b/roles/openshift_master/defaults/main.yml @@ -15,6 +15,12 @@ os_firewall_allow: port: 24224/tcp - service: Fluentd td-agent udp port: 24224/udp +- service: pcsd + port: 2224/tcp +- service: Corosync UDP + port: 5404/udp +- service: Corosync UDP + port: 5405/udp os_firewall_deny: - service: OpenShift api http port: 8080/tcp diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml index 6fd4dfb51..d57f9a4ea 100644 --- a/roles/openshift_master/handlers/main.yml +++ b/roles/openshift_master/handlers/main.yml @@ -1,3 +1,4 @@ --- - name: restart openshift-master service: name=openshift-master state=restarted + when: not openshift_master_ha diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 02905f32d..2311568dd 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -8,6 +8,10 @@ - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods when: openshift_master_oauth_grant_method is defined +- fail: + msg: "openshift_master_cluster_password must be set for multi-master installations" + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined + - name: Install OpenShift Master package yum: pkg=openshift-master state=present register: install_result @@ -16,6 +20,9 @@ openshift_facts: role: master local_facts: + cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}" + cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}" + cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}" debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}" api_port: "{{ openshift_master_api_port | default(None) }}" api_url: "{{ openshift_master_api_url | default(None) }}" @@ -114,12 +121,26 @@ - name: Start and enable openshift-master service: name=openshift-master enabled=yes state=started + when: not openshift_master_ha register: start_result - name: pause to prevent service restart from interfering with bootstrapping pause: seconds=30 when: start_result | changed +- name: Install cluster packagese + yum: pkg=pcs state=present + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool + register: install_result + +- name: Start and enable cluster service + service: name=pcsd enabled=yes state=started + when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool + +- name: Set the cluster user password + shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster + when: install_result | changed + - name: Create the OpenShift client config dir(s) file: path: "~{{ item }}/.kube" diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml index 8163ecd7f..03eb7e15f 100644 --- a/roles/openshift_master_ca/tasks/main.yml +++ b/roles/openshift_master_ca/tasks/main.yml @@ -14,7 +14,7 @@ - name: Create the master certificates if they do not already exist command: > {{ openshift.common.admin_binary }} create-master-certs - --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }} + --hostnames={{ openshift.common.all_hostnames | join(',') }} --master={{ openshift.master.api_url }} --public-master={{ openshift.master.public_api_url }} --cert-dir={{ openshift_master_config_dir }} --overwrite=false diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml index b5a3f8e40..297d53bcd 100644 --- a/roles/openshift_master_certificates/tasks/main.yml +++ b/roles/openshift_master_certificates/tasks/main.yml @@ -7,14 +7,20 @@ with_items: masters_needing_certs - file: - src: "{{ openshift_master_ca_cert }}" - dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt" - with_items: masters_needing_certs + src: "{{ openshift_master_config_dir }}/{{ item.1 }}" + dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}" + state: hard + with_nested: + - masters_needing_certs + - - ca.crt + - ca.key + - ca.serial.txt + - name: Create the master certificates if they do not already exist command: > {{ openshift.common.admin_binary }} create-master-certs - --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }} + --hostnames={{ item.openshift.common.all_hostnames | join(',') }} --master={{ item.openshift.master.api_url }} --public-master={{ item.openshift.master.public_api_url }} --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }} @@ -22,3 +28,5 @@ args: creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt" with_items: masters_needing_certs + + diff --git a/roles/openshift_master_certificates/vars/main.yml b/roles/openshift_master_certificates/vars/main.yml index 6e577b13b..6214f7918 100644 --- a/roles/openshift_master_certificates/vars/main.yml +++ b/roles/openshift_master_certificates/vars/main.yml @@ -1,6 +1,3 @@ --- openshift_generated_configs_dir: /etc/openshift/generated-configs openshift_master_config_dir: /etc/openshift/master -openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt" -openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key" -openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt" diff --git a/roles/openshift_master_cluster/README.md b/roles/openshift_master_cluster/README.md new file mode 100644 index 000000000..f150981fa --- /dev/null +++ b/roles/openshift_master_cluster/README.md @@ -0,0 +1,34 @@ +OpenShift Master Cluster +======================== + +TODO + +Requirements +------------ + +TODO + +Role Variables +-------------- + +TODO + +Dependencies +------------ + +TODO + +Example Playbook +---------------- + +TODO + +License +------- + +Apache License Version 2.0 + +Author Information +------------------ + +Jason DeTiberus (jdetiber@redhat.com) diff --git a/roles/openshift_master_cluster/meta/main.yml b/roles/openshift_master_cluster/meta/main.yml new file mode 100644 index 000000000..f3236e850 --- /dev/null +++ b/roles/openshift_master_cluster/meta/main.yml @@ -0,0 +1,16 @@ +--- +galaxy_info: + author: Jason DeTiberus + description: + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.8 + platforms: + - name: EL + versions: + - 7 + categories: + - cloud + - system +dependencies: +- { role: openshift_facts } diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml new file mode 100644 index 000000000..9a2546bca --- /dev/null +++ b/roles/openshift_master_cluster/tasks/configure.yml @@ -0,0 +1,43 @@ +--- +- fail: + msg: This role requires that openshift_master_cluster_vip is set + when: openshift_master_cluster_vip is not defined or not openshift_master_cluster_vip +- fail: + msg: This role requires that openshift_master_cluster_public_vip is set + when: openshift_master_cluster_public_vip is not defined or not openshift_master_cluster_public_vip + +- name: Authenticate to the cluster + command: pcs cluster auth -u hacluster -p {{ openshift_master_cluster_password }} {{ omc_cluster_hosts }} + +- name: Create the cluster + command: pcs cluster setup --name openshift_master {{ omc_cluster_hosts }} + +- name: Start the cluster + command: pcs cluster start --all + +- name: Enable the cluster on all nodes + command: pcs cluster enable --all + +- name: Set default resource stickiness + command: pcs resource defaults resource-stickiness=100 + +- name: Add the cluster VIP resource + command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_vip }} --group openshift-master + +- name: Add the cluster public VIP resource + command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_public_vip }} --group openshift-master + when: openshift_master_cluster_public_vip != openshift_master_cluster_vip + +- name: Add the cluster openshift-master service resource + command: pcs resource create master systemd:openshift-master --group openshift-master + +- name: Disable stonith + command: pcs property set stonith-enabled=false + +# TODO: handle case where api port is not 8443 +- name: Wait for the clustered master service to be available + wait_for: + host: "{{ openshift_master_cluster_vip }}" + port: 8443 + state: started + timeout: 300 diff --git a/roles/openshift_master_cluster/tasks/configure_deferred.yml b/roles/openshift_master_cluster/tasks/configure_deferred.yml new file mode 100644 index 000000000..a80b6c5b4 --- /dev/null +++ b/roles/openshift_master_cluster/tasks/configure_deferred.yml @@ -0,0 +1,8 @@ +--- +- debug: msg="Deferring config" + +- name: Start and enable openshift-master + service: + name: openshift-master + state: started + enabled: yes diff --git a/roles/openshift_master_cluster/tasks/main.yml b/roles/openshift_master_cluster/tasks/main.yml new file mode 100644 index 000000000..315947183 --- /dev/null +++ b/roles/openshift_master_cluster/tasks/main.yml @@ -0,0 +1,13 @@ +--- +- name: Test if cluster is already configured + command: pcs status + register: pcs_status + changed_when: false + failed_when: false + when: not openshift.master.cluster_defer_ha | bool + +- include: configure.yml + when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr" + +- include: configure_deferred.yml + when: openshift.master.cluster_defer_ha | bool diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml index 64a799dfb..c9f02aaf0 100644 --- a/roles/openshift_node_certificates/tasks/main.yml +++ b/roles/openshift_node_certificates/tasks/main.yml @@ -25,7 +25,7 @@ command: > {{ openshift.common.admin_binary }} create-server-cert --cert=server.crt --key=server.key --overwrite=true - --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname]|unique|join(",") }} + --hostnames={{ openshift.common.all_hostnames |join(",") }} --signer-cert={{ openshift_master_ca_cert }} --signer-key={{ openshift_master_ca_key }} --signer-serial={{ openshift_master_ca_serial }} -- cgit v1.2.3 From dc95d8e94f8866a6bc9c20bf60f1b91ec5a905c4 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 17 Jul 2015 12:38:30 -0400 Subject: Add explicit timeouts to ha master resource --- roles/openshift_master_cluster/tasks/configure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml index 9a2546bca..3510f9f62 100644 --- a/roles/openshift_master_cluster/tasks/configure.yml +++ b/roles/openshift_master_cluster/tasks/configure.yml @@ -29,7 +29,7 @@ when: openshift_master_cluster_public_vip != openshift_master_cluster_vip - name: Add the cluster openshift-master service resource - command: pcs resource create master systemd:openshift-master --group openshift-master + command: pcs resource create master systemd:openshift-master op start timeout=90s stop timeout=90s --group openshift-master - name: Disable stonith command: pcs property set stonith-enabled=false -- cgit v1.2.3 From 0a021c047bfb3fe5710be1e7de8cd577455f53c2 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Fri, 17 Jul 2015 15:02:03 -0400 Subject: fix typo --- roles/openshift_master/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 2311568dd..bb1689e5f 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -128,7 +128,7 @@ pause: seconds=30 when: start_result | changed -- name: Install cluster packagese +- name: Install cluster packages yum: pkg=pcs state=present when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool register: install_result -- cgit v1.2.3