From fbe3e47283e4ffc5966f707531dad00966803061 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Fri, 25 Aug 2017 16:33:54 -0400 Subject: Fix scaleup on containerized installations The output from member add had quotes added that caused etcd to fail. Also there were some places that needed to have retries added because containerized start/stop/restart are slower than rpm based installs. --- playbooks/common/openshift-etcd/scaleup.yml | 6 ++++-- roles/etcd_migrate/tasks/migrate.yml | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/playbooks/common/openshift-etcd/scaleup.yml b/playbooks/common/openshift-etcd/scaleup.yml index 52b90daca..a5567dc62 100644 --- a/playbooks/common/openshift-etcd/scaleup.yml +++ b/playbooks/common/openshift-etcd/scaleup.yml @@ -34,7 +34,7 @@ etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}" etcd_initial_cluster_state: "existing" - initial_etcd_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') }}" + initial_etcd_cluster: "{{ etcd_add_check.stdout_lines[3] | regex_replace('ETCD_INITIAL_CLUSTER=','') | regex_replace('\"','') }}" etcd_ca_setup: False r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}" - role: nickhammond.logrotate @@ -47,5 +47,7 @@ --ca-file {{ etcd_peer_ca_file }} -C {{ etcd_peer_url_scheme }}://{{ hostvars[etcd_ca_host].etcd_hostname }}:{{ etcd_client_port }} cluster-health - retries: 1 + register: scaleup_health + retries: 3 delay: 30 + until: scaleup_health.rc == 0 diff --git a/roles/etcd_migrate/tasks/migrate.yml b/roles/etcd_migrate/tasks/migrate.yml index 173de77f4..54a9c74ff 100644 --- a/roles/etcd_migrate/tasks/migrate.yml +++ b/roles/etcd_migrate/tasks/migrate.yml @@ -21,15 +21,24 @@ lineinfile: line: "ETCD_FORCE_NEW_CLUSTER=true" dest: /etc/etcd/etcd.conf + backup: true - name: Start etcd systemd: name: "{{ l_etcd_service }}" state: started +- name: Wait for cluster to become healthy after bringing up first member + command: > + etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} --endpoint https://{{ etcd_peer }}:{{ etcd_client_port }} cluster-health + register: l_etcd_migrate_health + until: l_etcd_migrate_health.rc == 0 + retries: 3 + delay: 30 - name: Unset ETCD_FORCE_NEW_CLUSTER=true on first etcd host lineinfile: line: "ETCD_FORCE_NEW_CLUSTER=true" dest: /etc/etcd/etcd.conf state: absent + backup: true - name: Restart first etcd host systemd: name: "{{ l_etcd_service }}" -- cgit v1.2.3