Merge pull request #4217 from ingvagabund/move-etcd-upgrade-code-into-role

Move etcd upgrade code into role

5 files changed, 176 insertions, 0 deletions

diff --git a/roles/etcd_upgrade/tasks/backup.yml b/roles/etcd_upgrade/tasks/backup.yml
new file mode 100644
index 000000000..1ea6fc59f
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/backup.yml
@@ -0,0 +1,71 @@
+---
+# INPUT r_etcd_backup_sufix_name
+# INPUT r_etcd_backup_tag
+# OUTPUT r_etcd_upgrade_backup_complete
+- set_fact:
+    # ORIGIN etcd_data_dir etcd_common.defaults
+    l_etcd_backup_dir: "{{ etcd_data_dir }}/openshift-backup-{{ r_etcd_backup_tag | default('') }}{{ r_etcd_backup_sufix_name }}"
+
+# TODO: replace shell module with command and update later checks
+- name: Check available disk space for etcd backup
+  shell: df --output=avail -k {{ etcd_data_dir }} | tail -n 1
+  register: avail_disk
+  # AUDIT:changed_when: `false` because we are only inspecting
+  # state, not manipulating anything
+  changed_when: false
+
+# TODO: replace shell module with command and update later checks
+- name: Check current etcd disk usage
+  shell: du --exclude='*openshift-backup*' -k {{ etcd_data_dir }} | tail -n 1 | cut -f1
+  register: etcd_disk_usage
+  when: r_etcd_upgrade_embedded_etcd | bool
+  # AUDIT:changed_when: `false` because we are only inspecting
+  # state, not manipulating anything
+  changed_when: false
+
+- name: Abort if insufficient disk space for etcd backup
+  fail:
+    msg: >
+      {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
+      {{ avail_disk.stdout }} Kb available.
+  when: (r_etcd_upgrade_embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
+
+# For non containerized and non embedded we should have the correct version of
+# etcd installed already. So don't do anything.
+#
+# For containerized installs we now exec into etcd_container
+#
+# For embedded non containerized we need to ensure we have the latest version
+# etcd on the host.
+- name: Install latest etcd for embedded
+  package:
+    name: etcd
+    state: latest
+  when:
+  - r_etcd_upgrade_embedded_etcd | bool
+  - not l_ostree_booted.stat.exists | bool
+
+- name: Generate etcd backup
+  command: >
+    {{ etcdctl_command }} backup --data-dir={{ etcd_data_dir }}
+    --backup-dir={{ l_etcd_backup_dir }}
+
+# According to the docs change you can simply copy snap/db
+# https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6
+- name: Check for v3 data store
+  stat:
+    path: "{{ etcd_data_dir }}/member/snap/db"
+  register: v3_db
+
+- name: Copy etcd v3 data store
+  command: >
+    cp -a {{ etcd_data_dir }}/member/snap/db
+    {{ l_etcd_backup_dir }}/member/snap/
+  when: v3_db.stat.exists
+
+- set_fact:
+    r_etcd_upgrade_backup_complete: True
+
+- name: Display location of etcd backup
+  debug:
+    msg: "Etcd backup created in {{ l_etcd_backup_dir }}"
diff --git a/roles/etcd_upgrade/tasks/main.yml b/roles/etcd_upgrade/tasks/main.yml
new file mode 100644
index 000000000..5178c14e3
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/main.yml
@@ -0,0 +1,14 @@
+---
+# INPUT r_etcd_upgrade_action
+- name: Fail if invalid etcd_upgrade_action provided
+  fail:
+    msg: "etcd_upgrade role can only be called with 'upgrade' or 'backup'"
+  when:
+  - r_etcd_upgrade_action not in ['upgrade', 'backup']
+
+- name: Detecting Atomic Host Operating System
+  stat:
+    path: /run/ostree-booted
+  register: l_ostree_booted
+
+- include: "{{ r_etcd_upgrade_action }}.yml"
diff --git a/roles/etcd_upgrade/tasks/upgrade.yml b/roles/etcd_upgrade/tasks/upgrade.yml
new file mode 100644
index 000000000..420c9638e
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/upgrade.yml
@@ -0,0 +1,11 @@
+---
+# INPUT r_etcd_upgrade_version
+# INPUT r_etcd_upgrade_mechanism
+- name: Failt if r_etcd_upgrade_mechanism is not set during upgrade
+  fail:
+    msg: "r_etcd_upgrade_mechanism can be only set to 'rpm' or 'image'"
+  when:
+  - r_etcd_upgrade_mechanism not in ['rpm', 'image']
+
+- name: "Upgrade {{ r_etcd_upgrade_mechanism }} based etcd"
+  include: upgrade_{{ r_etcd_upgrade_mechanism }}.yml
diff --git a/roles/etcd_upgrade/tasks/upgrade_image.yml b/roles/etcd_upgrade/tasks/upgrade_image.yml
new file mode 100644
index 000000000..136ec1142
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/upgrade_image.yml
@@ -0,0 +1,48 @@
+---
+# INPUT r_etcd_upgrade_version
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- name: Get current image
+  shell: "grep 'ExecStart=' {{ etcd_service_file }} | awk '{print $NF}'"
+  register: current_image
+
+- name: Set new_etcd_image
+  set_fact:
+    new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ r_etcd_upgrade_version ) }}"
+
+- name: Pull new etcd image
+  command: "docker pull {{ new_etcd_image }}"
+
+- name: Update to latest etcd image
+  replace:
+    dest: "{{ etcd_service_file }}"
+    regexp: "{{ current_image.stdout }}$"
+    replace: "{{ new_etcd_image }}"
+
+- name: Restart etcd_container
+  systemd:
+    name: "{{ etcd_service }}"
+    daemon_reload: yes
+    state: restarted
+
+## TODO: probably should just move this into the backup playbooks, also this
+## will fail on atomic host. We need to revisit how to do etcd backups there as
+## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
+- name: Upgrade etcd for etcdctl when not atomic
+  package: name=etcd state=latest
+  when: not l_ostree_booted.stat.exists | bool
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10
+
+- name: Store new etcd_image
+  # DEPENDENCY openshift_facts
+  openshift_facts:
+    role: etcd
+    local_facts:
+      etcd_image: "{{ new_etcd_image }}"
diff --git a/roles/etcd_upgrade/tasks/upgrade_rpm.yml b/roles/etcd_upgrade/tasks/upgrade_rpm.yml
new file mode 100644
index 000000000..324b69605
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/upgrade_rpm.yml
@@ -0,0 +1,32 @@
+---
+# INPUT r_etcd_upgrade_version?
+
+# F23 GA'd with etcd 2.0, currently has 2.2 in updates
+# F24 GA'd with etcd-2.2, currently has 2.2 in updates
+# F25 Beta currently has etcd 3.0
+# RHEL 7.3.4 with etcd-3.1.3-1.el7
+# RHEL 7.3.3 with etcd-3.1.0-2.el7
+# RHEL 7.3.2 with etcd-3.0.15-1.el7
+
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- set_fact:
+    l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}"
+
+- name: Update etcd RPM to {{ l_etcd_target_package }}
+  package:
+    name: "{{ l_etcd_target_package }}"
+    state: latest
+
+- name: Restart etcd
+  service:
+    name: "{{ etcd_service }}"
+    state: restarted
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10