From 0c31d72be3bf32f848eedad9859a81ba858f8c8f Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Wed, 1 Jun 2016 15:34:18 -0300 Subject: Docker 1.10 Upgrade Adds a separate playbook for Docker 1.10 upgrade that can be run standalone on a pre-existing 3.2 cluster. The upgrade will take each node out of rotation, and remove *all* containers and images on it, as this is reportedly faster and more storage efficient than performing the in place 1.10 upgrade. This process is integrated into the 3.1 to 3.2 upgrade process. Normal config playbooks now become 3.2 only, and require Docker 1.10. Users of older environments will have to use an appropriate openshift-ansible version. Config playbooks no longer are in the business of upgrading or downgrading docker. --- .../upgrades/docker/docker_upgrade.yml | 105 +++++++++++++++++++++ .../upgrades/docker/files/nuke_images.sh | 23 +++++ .../byo/openshift-cluster/upgrades/docker/roles | 1 + .../openshift-cluster/upgrades/docker/upgrade.yml | 29 ++++++ .../upgrades/v3_1_to_v3_2/upgrade.yml | 4 - 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml create mode 100644 playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh create mode 120000 playbooks/byo/openshift-cluster/upgrades/docker/roles create mode 100644 playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml (limited to 'playbooks/byo') diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml new file mode 100644 index 000000000..6c12e8245 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml @@ -0,0 +1,105 @@ + +- name: Check for appropriate Docker versions for 1.9.x to 1.10.x upgrade + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + roles: + - openshift_facts + tasks: + - name: Determine available Docker version + script: ../../../../common/openshift-cluster/upgrades/files/rpm_versions.sh docker + register: g_docker_version_result + when: not openshift.common.is_atomic | bool + + - name: Check if Docker is installed + command: rpm -q docker + register: pkg_check + failed_when: pkg_check.rc > 1 + changed_when: no + when: not openshift.common.is_atomic | bool + + - set_fact: + g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}" + when: not openshift.common.is_atomic | bool + + - name: Set fact if docker requires an upgrade + set_fact: + docker_upgrade: true + when: not openshift.common.is_atomic | bool and pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.10','<') + + - fail: + msg: This playbook requires access to Docker 1.10 or later + when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.10','<') + +# If a node fails, halt everything, the admin will need to clean up and we +# don't want to carry on, potentially taking out every node. The playbook can safely be re-run +# and will not take any action on a node already running 1.10+. +- name: Evacuate and upgrade nodes + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + serial: 1 + any_errors_fatal: true + tasks: + - debug: var=docker_upgrade + + - name: Prepare for Node evacuation + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + +# TODO: skip all node evac stuff for non-nodes (i.e. separate containerized etcd hosts) + - name: Evacuate Node for Kubelet upgrade + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --evacuate --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + + - name: Stop containerized services + service: name={{ item }} state=stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + - etcd + - openvswitch + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Remove all containers and images + script: files/nuke_images.sh docker + register: nuke_images_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Upgrade Docker + command: "{{ ansible_pkg_mgr}} update -y docker" + register: docker_upgrade_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_masters_to_config + + - name: Set node schedulability + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift.node.schedulable | bool + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config and openshift.node.schedulable | bool + diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh new file mode 100644 index 000000000..9a5ee2276 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Stop any running containers +running_container_count=`docker ps -q | wc -l` +if test $running_container_count -gt 0 +then + docker stop $(docker ps -q) +fi + +# Delete all containers +container_count=`docker ps -a -q | wc -l` +if test $container_count -gt 0 +then + docker rm -f -v $(docker ps -a -q) +fi + +# Delete all images (forcefully) +image_count=`docker images -q | wc -l` +if test $image_count -gt 0 +then + # Taken from: https://gist.github.com/brianclements/f72b2de8e307c7b56689#gistcomment-1443144 + docker rmi $(docker images | grep "$2/\|/$2 \| $2 \|$2 \|$2-\|$2_" | awk '{print $1 ":" $2}') 2>/dev/null || echo "No images matching \"$2\" left to purge." +fi diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/roles b/playbooks/byo/openshift-cluster/upgrades/docker/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/roles @@ -0,0 +1 @@ +../../../../../roles \ No newline at end of file diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml new file mode 100644 index 000000000..0f86abd89 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml @@ -0,0 +1,29 @@ +# Playbook to upgrade Docker to the max allowable version for an OpenShift cluster. +# +# Currently only supports upgrading 1.9.x to >= 1.10.x. +- hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + - add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: g_all_hosts | default([]) + changed_when: false + +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + +- include: ../../../../common/openshift-cluster/evaluate_groups.yml + vars: + # Do not allow adding hosts during upgrade. + g_new_master_hosts: [] + g_new_node_hosts: [] + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" + +- include: docker_upgrade.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml index 24617620b..8c89e118e 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml @@ -4,10 +4,6 @@ become: no gather_facts: no tasks: - - name: Verify Ansible version is greater than or equal to 1.9.4 and less than 2.0 - fail: - msg: "Unsupported ansible version: {{ ansible_version }} found." - when: ansible_version.full | version_compare('1.9.4', 'lt') or ansible_version.full | version_compare('2.0', 'ge') - include_vars: ../../../../byo/openshift-cluster/cluster_hosts.yml - add_host: name: "{{ item }}" -- cgit v1.2.3 From dcff138a3230a0f23cb627901b2a28cc6bec2e6b Mon Sep 17 00:00:00 2001 From: Devan Goodwin Date: Thu, 9 Jun 2016 08:37:34 -0300 Subject: Restore mistakenly reverted code. --- playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'playbooks/byo') diff --git a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml index 8c89e118e..24617620b 100644 --- a/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml +++ b/playbooks/byo/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml @@ -4,6 +4,10 @@ become: no gather_facts: no tasks: + - name: Verify Ansible version is greater than or equal to 1.9.4 and less than 2.0 + fail: + msg: "Unsupported ansible version: {{ ansible_version }} found." + when: ansible_version.full | version_compare('1.9.4', 'lt') or ansible_version.full | version_compare('2.0', 'ge') - include_vars: ../../../../byo/openshift-cluster/cluster_hosts.yml - add_host: name: "{{ item }}" -- cgit v1.2.3