From 6396343e2bc8559e5738e663b15a2c21bde37547 Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Tue, 8 Nov 2016 08:36:02 -0400 Subject: Fix etcd backup failure due to corrupted facts. Simplest way to reproduce this issue is to attempt to upgrade having removed /etc/ansible/facts.d/openshift.fact. Actual cause in the field is not entirely known but critically it is possible for embedded_etcd to default to true, causing the etcd fact lookup to check the wrong file and fail silently, resulting in no etcd_data_dir fact being set. --- .../openshift-cluster/upgrades/upgrade_control_plane.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'playbooks/common/openshift-cluster') diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index 927d9b4ca..afbfe0106 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -13,10 +13,23 @@ groups: etcd_hosts_to_backup with_items: "{{ groups.oo_etcd_to_config if groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config | length > 0 else groups.oo_first_master }}" +# If facts cache were for some reason deleted, this fact may not be set, and if not set +# it will always default to true. This causes problems for the etcd data dir fact detection +# so we must first make sure this is set correctly before attempting the backup. +- name: Set master embedded_etcd fact + hosts: oo_masters_to_config + roles: + - openshift_facts + tasks: + - openshift_facts: + role: master + local_facts: + embedded_etcd: "{{ groups.oo_etcd_to_config | length == 0 }}" + - name: Backup etcd hosts: etcd_hosts_to_backup vars: - embedded_etcd: "{{ hostvars[groups.oo_first_master.0].openshift.master.embedded_etcd }}" + embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" roles: - openshift_facts -- cgit v1.2.3