Merge pull request #4217 from ingvagabund/move-etcd-upgrade-code-into-role

Move etcd upgrade code into role

22 files changed, 270 insertions, 198 deletions

diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml
index 4fa7f9cdf..0d0ff798c 100644
--- a/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/ca.yml
@@ -9,7 +9,8 @@
 - name: Backup existing etcd CA certificate directories
   hosts: oo_etcd_to_config
   roles:
-  - etcd_common
+  - role: etcd_common
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   tasks:
   - name: Determine if CA certificate directory exists
     stat:
@@ -52,7 +53,8 @@
   vars:
     etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
   roles:
-  - etcd_common
+  - role: etcd_common
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   tasks:
   - name: Create a tarball of the etcd ca certs
     command: >
@@ -98,7 +100,8 @@
 - name: Retrieve etcd CA certificate
   hosts: oo_first_etcd
   roles:
-  - etcd_common
+  - role: etcd_common
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   tasks:
   - name: Retrieve etcd CA certificate
     fetch:
diff --git a/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml
index 2963a5940..6b5c805e6 100644
--- a/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml
+++ b/playbooks/common/openshift-cluster/redeploy-certificates/etcd.yml
@@ -3,7 +3,8 @@
   hosts: oo_first_etcd
   any_errors_fatal: true
   roles:
-    - etcd_common
+    - role: etcd_common
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   post_tasks:
     - name: Determine if generated etcd certificates exist
       stat:
@@ -27,7 +28,8 @@
   hosts: oo_etcd_to_config
   any_errors_fatal: true
   roles:
-    - etcd_common
+    - role: etcd_common
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   post_tasks:
     - name: Backup etcd certificates
       command: >
@@ -50,6 +52,7 @@
       etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
       etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
       openshift_ca_host: "{{ groups.oo_first_master.0 }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
 
 - name: Redeploy etcd client certificates for masters
   hosts: oo_masters_to_config
@@ -63,4 +66,5 @@
       etcd_cert_prefix: "master.etcd-"
       openshift_ca_host: "{{ groups.oo_first_master.0 }}"
       openshift_master_count: "{{ openshift.master.master_count | default(groups.oo_masters | length) }}"
+      r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
       when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
index 9d0333ca8..b7fd2c0c5 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml
@@ -1,84 +1,14 @@
 ---
 - name: Backup etcd
   hosts: oo_etcd_hosts_to_backup
-  vars:
-    embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
-    etcdctl_command: "{{ 'etcdctl' if not openshift.common.is_containerized or embedded_etcd else 'docker exec etcd_container etcdctl' if not openshift.common.is_etcd_system_container else 'runc exec etcd etcdctl' }}"
-    timestamp: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
   roles:
-  - openshift_facts
-  tasks:
-  # Ensure we persist the etcd role for this host in openshift_facts
-  - openshift_facts:
-      role: etcd
-      local_facts: {}
-    when: "'etcd' not in openshift"
-  - set_fact:
-      etcd_backup_dir: "{{ openshift.etcd.etcd_data_dir }}/openshift-backup-{{ backup_tag | default('') }}{{ timestamp }}"
-
-  # TODO: replace shell module with command and update later checks
-  - name: Check available disk space for etcd backup
-    shell: df --output=avail -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1
-    register: avail_disk
-    # AUDIT:changed_when: `false` because we are only inspecting
-    # state, not manipulating anything
-    changed_when: false
-
-  # TODO: replace shell module with command and update later checks
-  - name: Check current etcd disk usage
-    shell: du --exclude='*openshift-backup*' -k {{ openshift.etcd.etcd_data_dir }} | tail -n 1 | cut -f1
-    register: etcd_disk_usage
-    when: embedded_etcd | bool
-    # AUDIT:changed_when: `false` because we are only inspecting
-    # state, not manipulating anything
-    changed_when: false
-
-  - name: Abort if insufficient disk space for etcd backup
-    fail:
-      msg: >
-        {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
-        {{ avail_disk.stdout }} Kb available.
-    when: (embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
-
-  # For non containerized and non embedded we should have the correct version of
-  # etcd installed already. So don't do anything.
-  #
-  # For containerized installs we now exec into etcd_container
-  #
-  # For embedded non containerized we need to ensure we have the latest version
-  # etcd on the host.
-  - name: Install latest etcd for embedded
-    package:
-      name: etcd
-      state: latest
-    when:
-    - embedded_etcd | bool
-    - not openshift.common.is_atomic | bool
-
-  - name: Generate etcd backup
-    command: >
-      {{ etcdctl_command }} backup --data-dir={{ openshift.etcd.etcd_data_dir }}
-      --backup-dir={{ etcd_backup_dir }}
-
-  # According to the docs change you can simply copy snap/db
-  # https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6
-  - name: Check for v3 data store
-    stat:
-      path: "{{ openshift.etcd.etcd_data_dir }}/member/snap/db"
-    register: v3_db
-
-  - name: Copy etcd v3 data store
-    command: >
-      cp -a {{ openshift.etcd.etcd_data_dir }}/member/snap/db
-      {{ etcd_backup_dir }}/member/snap/
-    when: v3_db.stat.exists
-
-  - set_fact:
-      etcd_backup_complete: True
-
-  - name: Display location of etcd backup
-    debug:
-      msg: "Etcd backup created in {{ etcd_backup_dir }}"
+  - role: openshift_facts
+  - role: etcd_upgrade
+    r_etcd_upgrade_action: backup
+    r_etcd_backup_tag: etcd_backup_tag
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+    r_etcd_upgrade_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}"
+    r_etcd_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}"
 
 - name: Gate on etcd backup
   hosts: localhost
@@ -88,7 +18,7 @@
   - set_fact:
       etcd_backup_completed: "{{ hostvars
                                  | oo_select_keys(groups.oo_etcd_hosts_to_backup)
-                                 | oo_collect('inventory_hostname', {'etcd_backup_complete': true}) }}"
+                                 | oo_collect('inventory_hostname', {'r_etcd_upgrade_backup_complete': true}) }}"
   - set_fact:
       etcd_backup_failed: "{{ groups.oo_etcd_hosts_to_backup | difference(etcd_backup_completed) }}"
   - fail:
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml
deleted file mode 100644
index 30232110e..000000000
--- a/playbooks/common/openshift-cluster/upgrades/etcd/fedora_tasks.yml
+++ /dev/null
@@ -1,23 +0,0 @@
----
-# F23 GA'd with etcd 2.0, currently has 2.2 in updates
-# F24 GA'd with etcd-2.2, currently has 2.2 in updates
-# F25 Beta currently has etcd 3.0
-- name: Verify cluster is healthy pre-upgrade
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
-
-- name: Update etcd
-  package:
-    name: "etcd"
-    state: "latest"
-
-- name: Restart etcd
-  service:
-    name: etcd
-    state: restarted
-
-- name: Verify cluster is healthy
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0
-  retries: 3
-  delay: 10
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
index d9b59edcb..3e01883ae 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/main.yml
@@ -8,7 +8,7 @@
 - name: Backup etcd before upgrading anything
   include: backup.yml
   vars:
-    backup_tag: "pre-upgrade-"
+    etcd_backup_tag: "pre-upgrade-"
   when: openshift_etcd_backup | default(true) | bool
 
 - name: Drop etcdctl profiles
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml b/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml
deleted file mode 100644
index 3a972e8ab..000000000
--- a/playbooks/common/openshift-cluster/upgrades/etcd/rhel_tasks.yml
+++ /dev/null
@@ -1,20 +0,0 @@
----
-- name: Verify cluster is healthy pre-upgrade
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
-
-- name: Update etcd RPM
-  package:
-    name: etcd-{{ upgrade_version }}*
-    state: latest
-
-- name: Restart etcd
-  service:
-    name: etcd
-    state: restarted
-
-- name: Verify cluster is healthy
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
-  register: etcdctl
-  until: etcdctl.rc == 0
-  retries: 3
-  delay: 10
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml
index 54f9e21a1..0431c1ce0 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml
@@ -12,10 +12,10 @@
       # AUDIT:changed_when: `false` because we are only inspecting
       # state, not manipulating anything
       changed_when: false
-
     - debug:
         msg: "Etcd rpm version {{ etcd_rpm_version.stdout }} detected"
-    when: not openshift.common.is_containerized | bool
+    when:
+    - not openshift.common.is_containerized | bool
 
   - block:
     - name: Record containerized etcd version (docker)
@@ -54,84 +54,57 @@
 
     - debug:
         msg: "Etcd containerized version {{ etcd_container_version }} detected"
-
     when:
     - openshift.common.is_containerized | bool
 
-# I really dislike this copy/pasta but I wasn't able to find a way to get it to loop
-# through hosts, then loop through tasks only when appropriate
-- name: Upgrade to 2.1
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_rpm_members.yml
   vars:
-    upgrade_version: '2.1'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.1','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+    etcd_upgrade_version: '2.1'
 
-- name: Upgrade RPM hosts to 2.2
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_rpm_members.yml
   vars:
-    upgrade_version: '2.2'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.2','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+    etcd_upgrade_version: '2.2'
 
-- name: Upgrade containerized hosts to 2.2.5
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_image_members.yml
   vars:
-    upgrade_version: 2.2.5
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version | default('99') | version_compare('2.2','<') and openshift.common.is_containerized | bool
+    etcd_upgrade_version: '2.2.5'
 
-- name: Upgrade RPM hosts to 2.3
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_rpm_members.yml
   vars:
-    upgrade_version: '2.3'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('2.3','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+    etcd_upgrade_version: '2.3'
 
-- name: Upgrade containerized hosts to 2.3.7
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_image_members.yml
   vars:
-    upgrade_version: 2.3.7
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version | default('99') | version_compare('2.3','<') and openshift.common.is_containerized | bool
+    etcd_upgrade_version: '2.3.7'
 
-- name: Upgrade RPM hosts to 3.0
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_rpm_members.yml
   vars:
-    upgrade_version: '3.0'
-  tasks:
-  - include: rhel_tasks.yml
-    when: etcd_rpm_version.stdout | default('99') | version_compare('3.0','<') and ansible_distribution == 'RedHat' and not openshift.common.is_containerized | bool
+    etcd_upgrade_version: '3.0'
 
-- name: Upgrade containerized hosts to etcd3 image
-  hosts: oo_etcd_hosts_to_upgrade
-  serial: 1
+- include: upgrade_image_members.yml
   vars:
-    upgrade_version: 3.0.15
-  tasks:
-  - include: containerized_tasks.yml
-    when: etcd_container_version | default('99') | version_compare('3.0','<') and openshift.common.is_containerized | bool
+    etcd_upgrade_version: '3.0.15'
+
+- include: upgrade_rpm_members.yml
+  vars:
+    etcd_upgrade_version: '3.1'
+
+- include: upgrade_image_members.yml
+  vars:
+    etcd_upgrade_version: '3.1.3'
 
 - name: Upgrade fedora to latest
   hosts: oo_etcd_hosts_to_upgrade
   serial: 1
   tasks:
-  - include: fedora_tasks.yml
-    when: ansible_distribution == 'Fedora' and not openshift.common.is_containerized | bool
+  - include_role:
+      name: etcd_upgrade
+    when:
+    - ansible_distribution == 'Fedora'
+    - not openshift.common.is_containerized | bool
 
 - name: Backup etcd
   include: backup.yml
   vars:
-    backup_tag: "post-3.0-"
+    etcd_backup_tag: "post-3.0-"
   when: openshift_etcd_backup | default(true) | bool
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_image_members.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_image_members.yml
new file mode 100644
index 000000000..831ca8f57
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_image_members.yml
@@ -0,0 +1,17 @@
+---
+# INPUT etcd_upgrade_version
+# INPUT etcd_container_version
+# INPUT openshift.common.is_containerized
+- name: Upgrade containerized hosts to {{ etcd_upgrade_version }}
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  roles:
+  - role: etcd_upgrade
+    r_etcd_upgrade_action: upgrade
+    r_etcd_upgrade_mechanism: image
+    r_etcd_upgrade_version: "{{ etcd_upgrade_version }}"
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
+    etcd_peer: "{{ openshift.common.hostname }}"
+    when:
+    - etcd_container_version | default('99') | version_compare(etcd_upgrade_version,'<')
+    - openshift.common.is_containerized | bool
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_rpm_members.yml b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_rpm_members.yml
new file mode 100644
index 000000000..2e79451e0
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_rpm_members.yml
@@ -0,0 +1,18 @@
+---
+# INPUT etcd_upgrade_version
+# INPUT etcd_rpm_version
+# INPUT openshift.common.is_containerized
+- name: Upgrade to {{ etcd_upgrade_version }}
+  hosts: oo_etcd_hosts_to_upgrade
+  serial: 1
+  roles:
+  - role: etcd_upgrade
+    r_etcd_upgrade_action: upgrade
+    r_etcd_upgrade_mechanism: rpm
+    r_etcd_upgrade_version: "{{ etcd_upgrade_version }}"
+    r_etcd_common_etcd_runtime: "host"
+    etcd_peer: "{{ openshift.common.hostname }}"
+    when:
+    - etcd_rpm_version.stdout | default('99') | version_compare(etcd_upgrade_version, '<')
+    - ansible_distribution == 'RedHat'
+    - not openshift.common.is_containerized | bool
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 1b8106e0e..2cb6197d1 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -7,4 +7,5 @@
     etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}"
     etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}"
     etcd_certificates_etcd_hosts: "{{ groups.oo_etcd_to_config | default([], true) }}"
+    r_etcd_common_etcd_runtime: "{{ openshift.common.etcd_runtime }}"
   - role: nickhammond.logrotate
diff --git a/roles/etcd/defaults/main.yaml b/roles/etcd/defaults/main.yaml
index e45f53219..c0d1d5946 100644
--- a/roles/etcd/defaults/main.yaml
+++ b/roles/etcd/defaults/main.yaml
@@ -1,10 +1,4 @@
 ---
-etcd_service: "{{ 'etcd' if openshift.common.is_etcd_system_container | bool or not etcd_is_containerized | bool else 'etcd_container' }}"
-etcd_client_port: 2379
-etcd_peer_port: 2380
-etcd_url_scheme: http
-etcd_peer_url_scheme: http
-
 etcd_initial_cluster_state: new
 etcd_initial_cluster_token: etcd-cluster-1
 
diff --git a/roles/etcd_common/defaults/main.yml b/roles/etcd_common/defaults/main.yml
index d12e6a07f..e1a080b34 100644
--- a/roles/etcd_common/defaults/main.yml
+++ b/roles/etcd_common/defaults/main.yml
@@ -1,6 +1,9 @@
 ---
+# runc, docker, host
+r_etcd_common_etcd_runtime: "docker"
+
 # etcd server vars
-etcd_conf_dir: "{{ '/etc/etcd' if not openshift.common.is_etcd_system_container else '/var/lib/etcd/etcd.etcd/etc'  }}"
+etcd_conf_dir: "{{ '/etc/etcd' if r_etcd_common_etcd_runtime != 'runc' else '/var/lib/etcd/etcd.etcd/etc'  }}"
 etcd_system_container_conf_dir: /var/lib/etcd/etc
 etcd_conf_file: "{{ etcd_conf_dir }}/etcd.conf"
 etcd_ca_file: "{{ etcd_conf_dir }}/ca.crt"
@@ -38,3 +41,9 @@ etcd_is_thirdparty: False
 
 # etcd dir vars
 etcd_data_dir: /var/lib/etcd/
+
+# etcd ports and protocols
+etcd_client_port: 2379
+etcd_peer_port: 2380
+etcd_url_scheme: http
+etcd_peer_url_scheme: http
diff --git a/roles/etcd_common/vars/main.yml b/roles/etcd_common/vars/main.yml
new file mode 100644
index 000000000..00d697776
--- /dev/null
+++ b/roles/etcd_common/vars/main.yml
@@ -0,0 +1,4 @@
+---
+etcd_service: "{{ 'etcd_container' if r_etcd_common_etcd_runtime == 'docker' else 'etcd' }}"
+# Location of the service file is fixed and not meant to be changed
+etcd_service_file: "/etc/systemd/system/{{ etcd_service }}.service"
diff --git a/roles/etcd_upgrade/defaults/main.yml b/roles/etcd_upgrade/defaults/main.yml
new file mode 100644
index 000000000..01ad8a268
--- /dev/null
+++ b/roles/etcd_upgrade/defaults/main.yml
@@ -0,0 +1,9 @@
+---
+r_etcd_upgrade_action: upgrade
+r_etcd_upgrade_mechanism: rpm
+r_etcd_upgrade_embedded_etcd: False
+
+# etcd run on a host => use etcdctl command directly
+# etcd run as a docker container => use docker exec
+# etcd run as a runc container => use runc exec
+etcdctl_command: "{{ 'etcdctl' if r_etcd_common_etcd_runtime == 'host' or r_etcd_upgrade_embedded_etcd | bool else 'docker exec etcd_container etcdctl' if r_etcd_common_etcd_runtime == 'docker' else 'runc exec etcd etcdctl' }}"
diff --git a/roles/etcd_upgrade/meta/main.yml b/roles/etcd_upgrade/meta/main.yml
new file mode 100644
index 000000000..018bdc8d7
--- /dev/null
+++ b/roles/etcd_upgrade/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jan Chaloupka
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.9
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- role: etcd_common
diff --git a/roles/etcd_upgrade/tasks/backup.yml b/roles/etcd_upgrade/tasks/backup.yml
new file mode 100644
index 000000000..1ea6fc59f
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/backup.yml
@@ -0,0 +1,71 @@
+---
+# INPUT r_etcd_backup_sufix_name
+# INPUT r_etcd_backup_tag
+# OUTPUT r_etcd_upgrade_backup_complete
+- set_fact:
+    # ORIGIN etcd_data_dir etcd_common.defaults
+    l_etcd_backup_dir: "{{ etcd_data_dir }}/openshift-backup-{{ r_etcd_backup_tag | default('') }}{{ r_etcd_backup_sufix_name }}"
+
+# TODO: replace shell module with command and update later checks
+- name: Check available disk space for etcd backup
+  shell: df --output=avail -k {{ etcd_data_dir }} | tail -n 1
+  register: avail_disk
+  # AUDIT:changed_when: `false` because we are only inspecting
+  # state, not manipulating anything
+  changed_when: false
+
+# TODO: replace shell module with command and update later checks
+- name: Check current etcd disk usage
+  shell: du --exclude='*openshift-backup*' -k {{ etcd_data_dir }} | tail -n 1 | cut -f1
+  register: etcd_disk_usage
+  when: r_etcd_upgrade_embedded_etcd | bool
+  # AUDIT:changed_when: `false` because we are only inspecting
+  # state, not manipulating anything
+  changed_when: false
+
+- name: Abort if insufficient disk space for etcd backup
+  fail:
+    msg: >
+      {{ etcd_disk_usage.stdout }} Kb disk space required for etcd backup,
+      {{ avail_disk.stdout }} Kb available.
+  when: (r_etcd_upgrade_embedded_etcd | bool) and (etcd_disk_usage.stdout|int > avail_disk.stdout|int)
+
+# For non containerized and non embedded we should have the correct version of
+# etcd installed already. So don't do anything.
+#
+# For containerized installs we now exec into etcd_container
+#
+# For embedded non containerized we need to ensure we have the latest version
+# etcd on the host.
+- name: Install latest etcd for embedded
+  package:
+    name: etcd
+    state: latest
+  when:
+  - r_etcd_upgrade_embedded_etcd | bool
+  - not l_ostree_booted.stat.exists | bool
+
+- name: Generate etcd backup
+  command: >
+    {{ etcdctl_command }} backup --data-dir={{ etcd_data_dir }}
+    --backup-dir={{ l_etcd_backup_dir }}
+
+# According to the docs change you can simply copy snap/db
+# https://github.com/openshift/openshift-docs/commit/b38042de02d9780842dce95cfa0ef45d53b58bc6
+- name: Check for v3 data store
+  stat:
+    path: "{{ etcd_data_dir }}/member/snap/db"
+  register: v3_db
+
+- name: Copy etcd v3 data store
+  command: >
+    cp -a {{ etcd_data_dir }}/member/snap/db
+    {{ l_etcd_backup_dir }}/member/snap/
+  when: v3_db.stat.exists
+
+- set_fact:
+    r_etcd_upgrade_backup_complete: True
+
+- name: Display location of etcd backup
+  debug:
+    msg: "Etcd backup created in {{ l_etcd_backup_dir }}"
diff --git a/roles/etcd_upgrade/tasks/main.yml b/roles/etcd_upgrade/tasks/main.yml
new file mode 100644
index 000000000..5178c14e3
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/main.yml
@@ -0,0 +1,14 @@
+---
+# INPUT r_etcd_upgrade_action
+- name: Fail if invalid etcd_upgrade_action provided
+  fail:
+    msg: "etcd_upgrade role can only be called with 'upgrade' or 'backup'"
+  when:
+  - r_etcd_upgrade_action not in ['upgrade', 'backup']
+
+- name: Detecting Atomic Host Operating System
+  stat:
+    path: /run/ostree-booted
+  register: l_ostree_booted
+
+- include: "{{ r_etcd_upgrade_action }}.yml"
diff --git a/roles/etcd_upgrade/tasks/upgrade.yml b/roles/etcd_upgrade/tasks/upgrade.yml
new file mode 100644
index 000000000..420c9638e
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/upgrade.yml
@@ -0,0 +1,11 @@
+---
+# INPUT r_etcd_upgrade_version
+# INPUT r_etcd_upgrade_mechanism
+- name: Failt if r_etcd_upgrade_mechanism is not set during upgrade
+  fail:
+    msg: "r_etcd_upgrade_mechanism can be only set to 'rpm' or 'image'"
+  when:
+  - r_etcd_upgrade_mechanism not in ['rpm', 'image']
+
+- name: "Upgrade {{ r_etcd_upgrade_mechanism }} based etcd"
+  include: upgrade_{{ r_etcd_upgrade_mechanism }}.yml
diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml b/roles/etcd_upgrade/tasks/upgrade_image.yml
index 5f8b59e17..136ec1142 100644
--- a/playbooks/common/openshift-cluster/upgrades/etcd/containerized_tasks.yml
+++ b/roles/etcd_upgrade/tasks/upgrade_image.yml
@@ -1,27 +1,28 @@
 ---
+# INPUT r_etcd_upgrade_version
 - name: Verify cluster is healthy pre-upgrade
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+  command: "{{ etcdctlv2 }} cluster-health"
 
 - name: Get current image
-  shell: grep 'ExecStart=' /etc/systemd/system/etcd_container.service | awk '{print $NF}'
+  shell: "grep 'ExecStart=' {{ etcd_service_file }} | awk '{print $NF}'"
   register: current_image
 
 - name: Set new_etcd_image
   set_fact:
-    new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ upgrade_version ) }}"
+    new_etcd_image: "{{ current_image.stdout | regex_replace('/etcd.*$','/etcd:' ~ r_etcd_upgrade_version ) }}"
 
 - name: Pull new etcd image
   command: "docker pull {{ new_etcd_image }}"
 
 - name: Update to latest etcd image
   replace:
-    dest: /etc/systemd/system/etcd_container.service
+    dest: "{{ etcd_service_file }}"
     regexp: "{{ current_image.stdout }}$"
     replace: "{{ new_etcd_image }}"
 
 - name: Restart etcd_container
   systemd:
-    name: etcd_container
+    name: "{{ etcd_service }}"
     daemon_reload: yes
     state: restarted
 
@@ -30,16 +31,17 @@
 ## the container may be newer than etcdctl on the host. Assumes etcd3 obsoletes etcd (7.3.1)
 - name: Upgrade etcd for etcdctl when not atomic
   package: name=etcd state=latest
-  when: not openshift.common.is_atomic | bool
+  when: not l_ostree_booted.stat.exists | bool
 
 - name: Verify cluster is healthy
-  command: "etcdctl --cert-file /etc/etcd/peer.crt --key-file /etc/etcd/peer.key --ca-file /etc/etcd/ca.crt -C https://{{ openshift.common.hostname }}:2379 cluster-health"
+  command: "{{ etcdctlv2 }} cluster-health"
   register: etcdctl
   until: etcdctl.rc == 0
   retries: 3
   delay: 10
 
 - name: Store new etcd_image
+  # DEPENDENCY openshift_facts
   openshift_facts:
     role: etcd
     local_facts:
diff --git a/roles/etcd_upgrade/tasks/upgrade_rpm.yml b/roles/etcd_upgrade/tasks/upgrade_rpm.yml
new file mode 100644
index 000000000..324b69605
--- /dev/null
+++ b/roles/etcd_upgrade/tasks/upgrade_rpm.yml
@@ -0,0 +1,32 @@
+---
+# INPUT r_etcd_upgrade_version?
+
+# F23 GA'd with etcd 2.0, currently has 2.2 in updates
+# F24 GA'd with etcd-2.2, currently has 2.2 in updates
+# F25 Beta currently has etcd 3.0
+# RHEL 7.3.4 with etcd-3.1.3-1.el7
+# RHEL 7.3.3 with etcd-3.1.0-2.el7
+# RHEL 7.3.2 with etcd-3.0.15-1.el7
+
+- name: Verify cluster is healthy pre-upgrade
+  command: "{{ etcdctlv2 }} cluster-health"
+
+- set_fact:
+    l_etcd_target_package: "{{ 'etcd' if r_etcd_upgrade_version is not defined else 'etcd-'+r_etcd_upgrade_version+'*' }}"
+
+- name: Update etcd RPM to {{ l_etcd_target_package }}
+  package:
+    name: "{{ l_etcd_target_package }}"
+    state: latest
+
+- name: Restart etcd
+  service:
+    name: "{{ etcd_service }}"
+    state: restarted
+
+- name: Verify cluster is healthy
+  command: "{{ etcdctlv2 }} cluster-health"
+  register: etcdctl
+  until: etcdctl.rc == 0
+  retries: 3
+  delay: 10
diff --git a/roles/etcd_upgrade/vars/main.yml b/roles/etcd_upgrade/vars/main.yml
new file mode 100644
index 000000000..5ed919d42
--- /dev/null
+++ b/roles/etcd_upgrade/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# EXPECTS etcd_peer
+etcdctlv2: "etcdctl --cert-file {{ etcd_peer_cert_file }} --key-file {{ etcd_peer_key_file }} --ca-file {{ etcd_peer_ca_file }} -C https://{{ etcd_peer }}:{{ etcd_client_port }}"
diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml
index f657d86cf..1b9bda67e 100644
--- a/roles/openshift_facts/tasks/main.yml
+++ b/roles/openshift_facts/tasks/main.yml
@@ -15,6 +15,9 @@
     l_is_etcd_system_container: "{{ (use_etcd_system_container | default(use_system_containers) | bool) }}"
 - set_fact:
     l_any_system_container: "{{ l_is_etcd_system_container or l_is_openvswitch_system_container or l_is_node_system_container or l_is_master_system_container }}"
+- set_fact:
+    l_etcd_runtime: "{{ 'runc' if l_is_etcd_system_container else 'docker' if l_is_containerized else 'host' }}"
+
 
 - name: Validate python version
   fail:
@@ -80,6 +83,7 @@
       is_node_system_container: "{{ l_is_node_system_container | default(false) }}"
       is_master_system_container: "{{ l_is_master_system_container | default(false) }}"
       is_etcd_system_container: "{{ l_is_etcd_system_container | default(false) }}"
+      etcd_runtime: "{{ l_etcd_runtime }}"
       system_images_registry: "{{ system_images_registry | default('') }}"
       public_hostname: "{{ openshift_public_hostname | default(None) }}"
       public_ip: "{{ openshift_public_ip | default(None) }}"