Merge pull request #2005 from brenton/bz1338726

Bug 1338726 - never abort install if the latest version of docker is already installed

12 files changed, 202 insertions, 65 deletions

diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml
new file mode 100644
index 000000000..6c12e8245
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml
@@ -0,0 +1,105 @@
+
+- name: Check for appropriate Docker versions for 1.9.x to 1.10.x upgrade
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
+  roles:
+  - openshift_facts
+  tasks:
+  - name: Determine available Docker version
+    script: ../../../../common/openshift-cluster/upgrades/files/rpm_versions.sh docker
+    register: g_docker_version_result
+    when: not openshift.common.is_atomic | bool
+
+  - name: Check if Docker is installed
+    command: rpm -q docker
+    register: pkg_check
+    failed_when: pkg_check.rc > 1
+    changed_when: no
+    when: not openshift.common.is_atomic | bool
+
+  - set_fact:
+      g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}"
+    when: not openshift.common.is_atomic | bool
+
+  - name: Set fact if docker requires an upgrade
+    set_fact:
+      docker_upgrade: true
+    when: not openshift.common.is_atomic | bool and pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.10','<')
+
+  - fail:
+      msg: This playbook requires access to Docker 1.10 or later
+    when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.10','<')
+
+# If a node fails, halt everything, the admin will need to clean up and we
+# don't want to carry on, potentially taking out every node. The playbook can safely be re-run
+# and will not take any action on a node already running 1.10+.
+- name: Evacuate and upgrade nodes
+  hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
+  serial: 1
+  any_errors_fatal: true
+  tasks:
+  - debug: var=docker_upgrade
+
+  - name: Prepare for Node evacuation
+    command: >
+      {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=false
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config
+
+# TODO: skip all node evac stuff for non-nodes (i.e. separate containerized etcd hosts)
+  - name: Evacuate Node for Kubelet upgrade
+    command: >
+      {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --evacuate --force
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config
+
+  - name: Stop containerized services
+    service: name={{ item }} state=stopped
+    with_items:
+      - "{{ openshift.common.service_type }}-master"
+      - "{{ openshift.common.service_type }}-master-api"
+      - "{{ openshift.common.service_type }}-master-controllers"
+      - "{{ openshift.common.service_type }}-node"
+      - etcd
+      - openvswitch
+    failed_when: false
+    when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool
+
+  - name: Remove all containers and images
+    script: files/nuke_images.sh docker
+    register: nuke_images_result
+    when: docker_upgrade is defined and docker_upgrade | bool
+
+  - name: Upgrade Docker
+    command: "{{ ansible_pkg_mgr}} update -y docker"
+    register: docker_upgrade_result
+    when: docker_upgrade is defined and docker_upgrade | bool
+
+  - name: Restart containerized services
+    service: name={{ item }} state=started
+    with_items:
+      - etcd
+      - openvswitch
+      - "{{ openshift.common.service_type }}-master"
+      - "{{ openshift.common.service_type }}-master-api"
+      - "{{ openshift.common.service_type }}-master-controllers"
+      - "{{ openshift.common.service_type }}-node"
+    failed_when: false
+    when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool
+
+  - name: Wait for master API to come back online
+    become: no
+    local_action:
+      module: wait_for
+        host="{{ inventory_hostname }}"
+        state=started
+        delay=10
+        port="{{ openshift.master.api_port }}"
+    when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_masters_to_config
+
+  - name: Set node schedulability
+    command: >
+      {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=true
+    delegate_to: "{{ groups.oo_first_master.0 }}"
+    when: openshift.node.schedulable | bool
+    when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config and openshift.node.schedulable | bool
+
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh
new file mode 100644
index 000000000..9a5ee2276
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Stop any running containers
+running_container_count=`docker ps -q | wc -l`
+if test $running_container_count -gt 0
+then
+    docker stop $(docker ps -q)
+fi
+
+# Delete all containers
+container_count=`docker ps -a -q | wc -l`
+if test $container_count -gt 0
+then
+    docker rm -f -v $(docker ps -a -q)
+fi
+
+# Delete all images (forcefully)
+image_count=`docker images -q | wc -l`
+if test $image_count -gt 0
+then
+    # Taken from: https://gist.github.com/brianclements/f72b2de8e307c7b56689#gistcomment-1443144
+    docker rmi $(docker images | grep "$2/\|/$2 \| $2 \|$2 \|$2-\|$2_" | awk '{print $1 ":" $2}') 2>/dev/null || echo "No images matching \"$2\" left to purge."
+fi
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/roles b/playbooks/byo/openshift-cluster/upgrades/docker/roles
new file mode 120000
index 000000000..6bc1a7aef
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/docker/roles
@@ -0,0 +1 @@
+../../../../../roles
\ No newline at end of file
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml
new file mode 100644
index 000000000..0f86abd89
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml
@@ -0,0 +1,29 @@
+# Playbook to upgrade Docker to the max allowable version for an OpenShift cluster.
+#
+# Currently only supports upgrading 1.9.x to >= 1.10.x.
+- hosts: localhost
+  connection: local
+  become: no
+  gather_facts: no
+  tasks:
+  - include_vars: ../../cluster_hosts.yml
+  - add_host:
+      name: "{{ item }}"
+      groups: l_oo_all_hosts
+    with_items: g_all_hosts | default([])
+    changed_when: false
+
+- hosts: l_oo_all_hosts
+  gather_facts: no
+  tasks:
+  - include_vars: ../../cluster_hosts.yml
+
+- include: ../../../../common/openshift-cluster/evaluate_groups.yml
+  vars:
+    # Do not allow adding hosts during upgrade.
+    g_new_master_hosts: []
+    g_new_node_hosts: []
+    openshift_cluster_id: "{{ cluster_id | default('default') }}"
+    openshift_deployment_type: "{{ deployment_type }}"
+
+- include: docker_upgrade.yml
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml
deleted file mode 100644
index 9ade795f2..000000000
--- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-- name: Check if Docker is installed
-  command: rpm -q docker
-  register: pkg_check
-  failed_when: pkg_check.rc > 1
-  changed_when: no
-
-- name: Upgrade Docker
-  command: "{{ ansible_pkg_mgr}} update -y docker"
-  when: pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.9','<')
-  register: docker_upgrade
-
-- name: Restart Docker
-  command: systemctl restart docker
-  when: docker_upgrade | changed
-
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
index c93bf2a17..156e80c0f 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
@@ -3,13 +3,13 @@
 # The restart playbook should be run after this playbook completes.
 ###############################################################################
 
-- name: Upgrade docker
+- include: ../../../../byo/openshift-cluster/upgrades/docker/docker_upgrade.yml
+
+- name: Update Docker facts
   hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config
   roles:
   - openshift_facts
   tasks:
-  - include: docker_upgrade.yml
-    when: not openshift.common.is_atomic | bool
   - name: Set post docker install facts
     openshift_facts:
       role: "{{ item.role }}"
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 5f94500a0..3368e7cd9 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -1,41 +1,42 @@
 ---
-# tasks file for docker
-
-- name: Get current installed version if docker_version is specified
+# Going forward we require Docker 1.10 or greater. If the user has a lesser version installed they must run a separate upgrade process.
+- name: Get current installed Docker version
   command: "{{ repoquery_cmd }} --installed --qf '%{version}' docker"
-  when:  not openshift.common.is_atomic | bool and docker_version != ''
-  register: docker_version_result
+  when:  not openshift.common.is_atomic | bool
+  register: curr_docker_version
   changed_when: false
 
-- stat: path=/etc/sysconfig/docker-storage
-  register: docker_storage_check
+# TODO: The use of upgrading var will be removed in the coming upgrade refactor. This is a temporary
+# fix to wory around the fact that right now, this role is called during upgrade, before we're
+# ready to upgrade Docker.
+- name: Fail if Docker upgrade is required
+  fail:
+    msg: "Docker {{ curr_docker_version.stdout }} must be upgraded to Docker 1.10 or greater"
+  when: not upgrading | bool and not curr_docker_version | skipped and curr_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<')
 
-- name: Remove deferred deletion for downgrades from 1.9
+- name: Get latest available version of Docker
   command: >
-    sed -i 's/--storage-opt dm.use_deferred_deletion=true//' /etc/sysconfig/docker-storage
-  when: docker_storage_check.stat.exists | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<')
-
-- name: Downgrade docker if necessary
-  command: "{{ ansible_pkg_mgr }} swap -y docker-* docker-*{{ docker_version }}"
-  register: docker_downgrade_result
-  when: not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'gt')
+    {{ repoquery_cmd }} --qf '%{version}' "docker"
+  register: avail_docker_version
+  failed_when: false
+  changed_when: false
+  when: not curr_docker_version.stdout | default('0.0', True) | version_compare('1.10', '>=') and not openshift.common.is_atomic | bool
 
-- name: Install docker
-  action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version if docker_version is defined and docker_version != '' else '' }} state=present"
-  when: not openshift.common.is_atomic | bool and docker_downgrade_result | skipped
+- name: Verify Docker >= 1.10 is available
+  fail:
+    msg: "Docker {{ avail_docker_version.stdout }} is available, but 1.10 or greater is required"
+  when: not avail_docker_version | skipped and avail_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<')
 
-# If docker were enabled and started before we downgraded it may have entered a
-# failed state. Check for that and clear it if necessary.
-- name: Check that docker hasn't entered failed state
-  command: systemctl show docker
-  register: docker_state
-  changed_when: False
+- stat: path=/etc/sysconfig/docker-storage
+  register: docker_storage_check
 
-- name: Reset docker service state
-  command: systemctl reset-failed docker.service
-  when: " 'ActiveState=failed' in docker_state.stdout "
+# Make sure Docker is installed, but does not update a running version.
+# Docker upgrades are handled by a separate playbook.
+- name: Install Docker
+  action: "{{ ansible_pkg_mgr }} name=docker state=present"
+  when: not openshift.common.is_atomic | bool
 
-- name: Start the docker service
+- name: Start the Docker service
   service:
     name: docker
     enabled: yes
@@ -87,7 +88,7 @@
     - restart docker
   when: "{{ 'http_proxy' in openshift.common or 'https_proxy' in openshift.common and docker_check.stat.isreg }}"
 
-- name: Set various docker options
+- name: Set various Docker options
   lineinfile:
     dest: /etc/sysconfig/docker
     regexp: '^OPTIONS=.*$'
diff --git a/roles/openshift_cli/tasks/main.yml b/roles/openshift_cli/tasks/main.yml
index bfa60e5b0..c0a712513 100644
--- a/roles/openshift_cli/tasks/main.yml
+++ b/roles/openshift_cli/tasks/main.yml
@@ -5,7 +5,7 @@
 
 - name: Pull CLI Image
   command: >
-    docker pull {{ openshift.common.cli_image }}:{{ openshift_version }}
+    docker pull {{ openshift.common.cli_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }}
   when: openshift.common.is_containerized | bool
 
 - name: Create /usr/local/bin/openshift cli wrapper
diff --git a/roles/openshift_cli/templates/openshift.j2 b/roles/openshift_cli/templates/openshift.j2
index 437e08aab..8a3f3a257 100644
--- a/roles/openshift_cli/templates/openshift.j2
+++ b/roles/openshift_cli/templates/openshift.j2
@@ -5,14 +5,14 @@ fi
 cmd=`basename $0`
 user=`id -u`
 group=`id -g`
-image_tag={{ openshift_version }}
+image_tag="{{ openshift_version }}"
 
 >&2 echo """
 ================================================================================
 ATTENTION: You are running ${cmd} via a wrapper around 'docker run {{ openshift.common.cli_image }}:${image_tag}'.
 This wrapper is intended only to be used to bootstrap an environment. Please
 install client tools on another host once you have granted cluster-admin
-privileges to a user. 
+privileges to a user.
 {% if openshift.common.deployment_type in ['openshift-enterprise','atomic-enterprise'] %}
 See https://docs.openshift.com/enterprise/latest/cli_reference/get_started_cli.html
 {% else %}
@@ -21,4 +21,8 @@ See https://docs.openshift.org/latest/cli_reference/get_started_cli.html
 =================================================================================
 """
 
-docker run -i --privileged --net=host --user=${user}:${group} -v ~/.kube:/root/.kube -v /tmp:/tmp -v {{ openshift.common.config_base}}:{{ openshift.common.config_base }} -e KUBECONFIG=/root/.kube/config --entrypoint ${cmd} --rm {{ openshift.common.cli_image }}:${image_tag} "${@}"
+if [ -n "$image_tag" ]; then
+    image_tag=":$image_tag"
+fi
+
+docker run -i --privileged --net=host --user=${user}:${group} -v ~/.kube:/root/.kube -v /tmp:/tmp -v {{ openshift.common.config_base}}:{{ openshift.common.config_base }} -e KUBECONFIG=/root/.kube/config --entrypoint ${cmd} --rm {{ openshift.common.cli_image }}${image_tag} "${@}"
diff --git a/roles/openshift_docker_facts/tasks/main.yml b/roles/openshift_docker_facts/tasks/main.yml
index 43359dcb5..cdea90413 100644
--- a/roles/openshift_docker_facts/tasks/main.yml
+++ b/roles/openshift_docker_facts/tasks/main.yml
@@ -57,14 +57,3 @@
     l_common_version: "{{ common_version.stdout | default('0.0', True) }}"
   when: not openshift.common.is_containerized | bool
 
-- name: Set docker version to be installed
-  set_fact:
-    docker_version: "{{ '1.8.2' }}"
-  when: " ( l_common_version | version_compare('3.2','<') and openshift.common.service_type in ['openshift', 'atomic-openshift'] ) or
-          ( l_common_version | version_compare('1.1.4','<') and openshift.common.service_type == 'origin' )"
-
-- name: Set docker version to be installed
-  set_fact:
-    docker_version: "{{ '1.9.1' }}"
-  when: " ( l_common_version | version_compare('3.2','>') and openshift.common.service_type == 'atomic-openshift' ) or
-          ( l_common_version | version_compare('1.2','>') and openshift.common.service_type == 'origin' )"
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 52ea8bccf..28faee155 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -29,7 +29,7 @@
 
 - name: Pull master image
   command: >
-    docker pull {{ openshift.master.master_image }}:{{ openshift_version }}
+    docker pull {{ openshift.master.master_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }}
   when: openshift.common.is_containerized | bool
 
 - name: Create openshift.common.data_dir
diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml
index e8bd13855..657e99e87 100644
--- a/roles/openshift_node/tasks/main.yml
+++ b/roles/openshift_node/tasks/main.yml
@@ -39,12 +39,12 @@
 
 - name: Pull node image
   command: >
-    docker pull {{ openshift.node.node_image }}:{{ openshift_version }}
+    docker pull {{ openshift.node.node_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }}
   when: openshift.common.is_containerized | bool
 
 - name: Pull OpenVSwitch image
   command: >
-    docker pull {{ openshift.node.ovs_image }}:{{ openshift_version }}
+    docker pull {{ openshift.node.ovs_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }}
   when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool
 
 - name: Install the systemd units