Merge pull request #5672 from ingvagabund/migrate-embedded-etcd

Automatic merge from submit-queue.

Migrate embedded etcd to external etcd

Trello card: https://trello.com/c/9fnBfkT5/528-migrate-embedded-etcd-hosts-to-external-process?menu=filter&filter=label:committed-3.7

Tested on:
- [x] embedded etcd -> external rpm etcd
- [x] embedded etcd -> external docker etcd

One needs to set the `openshift_image_tag` before running the migration to containerized docker etcd.

4 files changed, 203 insertions, 27 deletions

diff --git a/playbooks/common/openshift-etcd/certificates.yml b/playbooks/common/openshift-etcd/certificates.yml
index 31a0f50d8..eb6b94f33 100644
--- a/playbooks/common/openshift-etcd/certificates.yml
+++ b/playbooks/common/openshift-etcd/certificates.yml
@@ -1,29 +1,4 @@
 ---
-- name: Create etcd server certificates for etcd hosts
-  hosts: oo_etcd_to_config
-  any_errors_fatal: true
-  roles:
-    - role: openshift_etcd_facts
-  post_tasks:
-    - include_role:
-        name: etcd
-        tasks_from: server_certificates
-      vars:
-        etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
-        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
-        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+- include: server_certificates.yml
 
-- name: Create etcd client certificates for master hosts
-  hosts: oo_masters_to_config
-  any_errors_fatal: true
-  roles:
-    - role: openshift_etcd_facts
-    - role: openshift_etcd_client_certificates
-      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
-      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
-      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
-      etcd_cert_prefix: "master.etcd-"
-      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
-      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
-      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+- include: master_etcd_certificates.yml
diff --git a/playbooks/common/openshift-etcd/embedded2external.yml b/playbooks/common/openshift-etcd/embedded2external.yml
new file mode 100644
index 000000000..9264f3c32
--- /dev/null
+++ b/playbooks/common/openshift-etcd/embedded2external.yml
@@ -0,0 +1,172 @@
+---
+- name: Pre-migrate checks
+  hosts: localhost
+  tasks:
+  # Check there is only one etcd host
+  - assert:
+      that: groups.oo_etcd_to_config | default([]) | length == 1
+      msg: "[etcd] group must contain only one host"
+  # Check there is only one master
+  - assert:
+      that: groups.oo_masters_to_config | default([]) | length == 1
+      msg: "[master] group must contain only one host"
+
+# 1. stop a master
+- name: Prepare masters for etcd data migration
+  hosts: oo_first_master
+  roles:
+  - role: openshift_facts
+  tasks:
+  - name: Check the master API is ready
+    include_role:
+      name: openshift_master
+      tasks_from: check_master_api_is_ready
+  - set_fact:
+      master_service: "{{ openshift.common.service_type + '-master' }}"
+      embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
+  - debug:
+      msg: "master service name: {{ master_service }}"
+  - name: Stop master
+    service:
+      name: "{{ master_service }}"
+      state: stopped
+  # 2. backup embedded etcd
+  # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285
+  - include_role:
+      name: etcd
+      tasks_from: backup
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.archive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ embedded_etcd_backup_suffix }}"
+
+# 3. deploy certificates (for etcd and master)
+- include: ca.yml
+
+- include: server_certificates.yml
+
+- name: Backup etcd client certificates for master host
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup_master_etcd_certificates
+
+- name: Redeploy master etcd certificates
+  include: master_etcd_certificates.yml
+  vars:
+    etcd_certificates_redeploy: "{{ true }}"
+
+# 4. deploy external etcd
+- include: ../openshift-etcd/config.yml
+
+# 5. stop external etcd
+- name: Cleanse etcd
+  hosts: oo_etcd_to_config[0]
+  gather_facts: no
+  pre_tasks:
+  - include_role:
+      name: etcd
+      tasks_from: disable_etcd
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+  - include_role:
+      name: etcd
+      tasks_from: clean_data
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+
+# 6. copy the embedded etcd backup to the external host
+# TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory
+- name: Copy embedded etcd backup to the external host
+  hosts: localhost
+  tasks:
+  - name: Create local temp directory for syncing etcd backup
+    local_action: command mktemp -d /tmp/etcd_backup-XXXXXXX
+    register: g_etcd_client_mktemp
+    changed_when: False
+    become: no
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.fetch
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_first_master.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_embedded_etcd: "{{ true }}"
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_first_master[0] }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.copy
+    vars:
+      r_etcd_common_etcd_runtime: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.etcd_runtime }}"
+      etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+    delegate_to: "{{ groups.oo_etcd_to_config[0] }}"
+
+  - debug:
+      msg: "etcd_backup_dest_directory: {{ g_etcd_client_mktemp.stdout }}"
+
+  - name: Delete temporary directory
+    local_action: file path="{{ g_etcd_client_mktemp.stdout }}" state=absent
+    changed_when: False
+    become: no
+
+# 7. force new cluster from the backup
+- name: Force new etcd cluster
+  hosts: oo_etcd_to_config[0]
+  tasks:
+  - include_role:
+      name: etcd
+      tasks_from: backup.unarchive
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+
+  - include_role:
+      name: etcd
+      tasks_from: backup.force_new_cluster
+    vars:
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      r_etcd_common_backup_tag: pre-migrate
+      r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}"
+      etcd_peer: "{{ openshift.common.ip }}"
+      etcd_url_scheme: "https"
+      etcd_peer_url_scheme: "https"
+
+# 8. re-configure master to use the external etcd
+- name: Configure master to use external etcd
+  hosts: oo_first_master
+  tasks:
+  - include_role:
+      name: openshift_master
+      tasks_from: configure_external_etcd
+    vars:
+      etcd_peer_url_scheme: "https"
+      etcd_ip: "{{ openshift.common.ip }}"
+      etcd_peer_port: 2379
+
+  # 9. start the master
+  - name: Start master
+    service:
+      name: "{{ master_service }}"
+      state: started
+    register: service_status
+    until: service_status.state is defined and service_status.state == "started"
+    retries: 5
+    delay: 10
diff --git a/playbooks/common/openshift-etcd/master_etcd_certificates.yml b/playbooks/common/openshift-etcd/master_etcd_certificates.yml
new file mode 100644
index 000000000..0a25aac57
--- /dev/null
+++ b/playbooks/common/openshift-etcd/master_etcd_certificates.yml
@@ -0,0 +1,14 @@
+---
+- name: Create etcd client certificates for master hosts
+  hosts: oo_masters_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+    - role: openshift_etcd_client_certificates
+      etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+      etcd_cert_subdir: "openshift-master-{{ openshift.common.hostname }}"
+      etcd_cert_config_dir: "{{ openshift.common.config_base }}/master"
+      etcd_cert_prefix: "master.etcd-"
+      openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+      when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
diff --git a/playbooks/common/openshift-etcd/server_certificates.yml b/playbooks/common/openshift-etcd/server_certificates.yml
new file mode 100644
index 000000000..10e06747b
--- /dev/null
+++ b/playbooks/common/openshift-etcd/server_certificates.yml
@@ -0,0 +1,15 @@
+---
+- name: Create etcd server certificates for etcd hosts
+  hosts: oo_etcd_to_config
+  any_errors_fatal: true
+  roles:
+    - role: openshift_etcd_facts
+  post_tasks:
+    - include_role:
+        name: etcd
+        tasks_from: server_certificates
+      vars:
+        etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
+        etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+        r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"