From dea9abfe22864cf10d85d85370b1633ca18060b6 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Mon, 14 Dec 2015 15:29:24 -0400 Subject: Implement simple master rolling restarts. Blocks running ansible on a host that will be restarted. Can restart just services, or optionally the full system. --- playbooks/byo/openshift-cluster/restart.yml | 7 +++ playbooks/common/openshift-cluster/restart.yml | 78 ++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 playbooks/byo/openshift-cluster/restart.yml create mode 100644 playbooks/common/openshift-cluster/restart.yml (limited to 'playbooks') diff --git a/playbooks/byo/openshift-cluster/restart.yml b/playbooks/byo/openshift-cluster/restart.yml new file mode 100644 index 000000000..da0da69a6 --- /dev/null +++ b/playbooks/byo/openshift-cluster/restart.yml @@ -0,0 +1,7 @@ +--- +- include: ../../common/openshift-cluster/restart.yml + vars: + g_etcd_hosts: "{{ groups.etcd | default([]) }}" + g_master_hosts: "{{ groups.masters | default([]) }}" + g_node_hosts: "{{ groups.nodes | default([]) }}" + g_lb_hosts: "{{ groups.lb | default([]) }}" diff --git a/playbooks/common/openshift-cluster/restart.yml b/playbooks/common/openshift-cluster/restart.yml new file mode 100644 index 000000000..4117f7297 --- /dev/null +++ b/playbooks/common/openshift-cluster/restart.yml @@ -0,0 +1,78 @@ +--- +- include: evaluate_groups.yml +# TODO: verify this is an HA environment +# TODO: fork for pacemaker vs haproxy (based on?) + +- name: Validate configuration for rolling restart + hosts: oo_masters_to_config + tasks: + - set_fact: + openshift_rolling_restart_mode: "{{ openshift_rolling_restart_mode | default('services') }}" + - fail: + msg: "openshift_rolling_restart_mode must be set to either 'services' or 'system'" + when: openshift_rolling_restart_mode is defined and openshift_rolling_restart_mode not in ["services", "system"] + +# Creating a temp file on localhost, we then check each system that will +# be rebooted to see if that file exists, if so we know we're running +# ansible on a machine that needs a reboot, and we need to error out. +- name: Create temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - local_action: command mktemp + register: mktemp + changed_when: False + +- name: Check if temp file exists on any masters + hosts: oo_masters_to_config + tasks: + - stat: path="{{ hostvars.localhost.mktemp.stdout }}" + register: exists + +- name: Cleanup temp file on localhost + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - file: path="{{ hostvars.localhost.mktemp.stdout }}" state=absent + +- name: Fail if restarting the system where ansible is running + hosts: oo_masters_to_config + any_errors_fatal: true + tasks: + - fail: msg="Cannot run playbook on a host that will be restarted." + when: exists.stat.exists + +- name: Restart Masters + hosts: oo_masters_to_config + serial: 1 + roles: + - openshift_facts + tasks: + - name: Restart master system + # https://github.com/ansible/ansible/issues/10616 + shell: sleep 2 && shutdown -r now "OpenShift Ansible master rolling restart" + async: 1 + poll: 0 + ignore_errors: true + become: yes + when: openshift_rolling_restart_mode == 'system' + - name: Restart master services + service: + name: "{{ openshift.common.service_type }}-master-api" + state: restarted + # NOTE: no need to check openshift_master_ha here, we know it must be, + # thus the api service is the one we restart. + when: openshift_rolling_restart_mode == 'services' + + - name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port=8443 # TODO: should this be made a master host variable? -- cgit v1.2.3