diff options
12 files changed, 202 insertions, 65 deletions
diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml new file mode 100644 index 000000000..6c12e8245 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/docker_upgrade.yml @@ -0,0 +1,105 @@ + +- name: Check for appropriate Docker versions for 1.9.x to 1.10.x upgrade + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + roles: + - openshift_facts + tasks: + - name: Determine available Docker version + script: ../../../../common/openshift-cluster/upgrades/files/rpm_versions.sh docker + register: g_docker_version_result + when: not openshift.common.is_atomic | bool + + - name: Check if Docker is installed + command: rpm -q docker + register: pkg_check + failed_when: pkg_check.rc > 1 + changed_when: no + when: not openshift.common.is_atomic | bool + + - set_fact: + g_docker_version: "{{ g_docker_version_result.stdout | from_yaml }}" + when: not openshift.common.is_atomic | bool + + - name: Set fact if docker requires an upgrade + set_fact: + docker_upgrade: true + when: not openshift.common.is_atomic | bool and pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.10','<') + + - fail: + msg: This playbook requires access to Docker 1.10 or later + when: g_docker_version.avail_version | default(g_docker_version.curr_version, true) | version_compare('1.10','<') + +# If a node fails, halt everything, the admin will need to clean up and we +# don't want to carry on, potentially taking out every node. The playbook can safely be re-run +# and will not take any action on a node already running 1.10+. +- name: Evacuate and upgrade nodes + hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config + serial: 1 + any_errors_fatal: true + tasks: + - debug: var=docker_upgrade + + - name: Prepare for Node evacuation + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=false + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + +# TODO: skip all node evac stuff for non-nodes (i.e. separate containerized etcd hosts) + - name: Evacuate Node for Kubelet upgrade + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --evacuate --force + delegate_to: "{{ groups.oo_first_master.0 }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config + + - name: Stop containerized services + service: name={{ item }} state=stopped + with_items: + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + - etcd + - openvswitch + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Remove all containers and images + script: files/nuke_images.sh docker + register: nuke_images_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Upgrade Docker + command: "{{ ansible_pkg_mgr}} update -y docker" + register: docker_upgrade_result + when: docker_upgrade is defined and docker_upgrade | bool + + - name: Restart containerized services + service: name={{ item }} state=started + with_items: + - etcd + - openvswitch + - "{{ openshift.common.service_type }}-master" + - "{{ openshift.common.service_type }}-master-api" + - "{{ openshift.common.service_type }}-master-controllers" + - "{{ openshift.common.service_type }}-node" + failed_when: false + when: docker_upgrade is defined and docker_upgrade | bool and openshift.common.is_containerized | bool + + - name: Wait for master API to come back online + become: no + local_action: + module: wait_for + host="{{ inventory_hostname }}" + state=started + delay=10 + port="{{ openshift.master.api_port }}" + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_masters_to_config + + - name: Set node schedulability + command: > + {{ openshift.common.admin_binary }} manage-node {{ openshift.common.hostname | lower }} --schedulable=true + delegate_to: "{{ groups.oo_first_master.0 }}" + when: openshift.node.schedulable | bool + when: docker_upgrade is defined and docker_upgrade | bool and inventory_hostname in groups.oo_nodes_to_config and openshift.node.schedulable | bool + diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh new file mode 100644 index 000000000..9a5ee2276 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/files/nuke_images.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Stop any running containers +running_container_count=`docker ps -q | wc -l` +if test $running_container_count -gt 0 +then + docker stop $(docker ps -q) +fi + +# Delete all containers +container_count=`docker ps -a -q | wc -l` +if test $container_count -gt 0 +then + docker rm -f -v $(docker ps -a -q) +fi + +# Delete all images (forcefully) +image_count=`docker images -q | wc -l` +if test $image_count -gt 0 +then + # Taken from: https://gist.github.com/brianclements/f72b2de8e307c7b56689#gistcomment-1443144 + docker rmi $(docker images | grep "$2/\|/$2 \| $2 \|$2 \|$2-\|$2_" | awk '{print $1 ":" $2}') 2>/dev/null || echo "No images matching \"$2\" left to purge." +fi diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/roles b/playbooks/byo/openshift-cluster/upgrades/docker/roles new file mode 120000 index 000000000..6bc1a7aef --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/roles @@ -0,0 +1 @@ +../../../../../roles
\ No newline at end of file diff --git a/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml new file mode 100644 index 000000000..0f86abd89 --- /dev/null +++ b/playbooks/byo/openshift-cluster/upgrades/docker/upgrade.yml @@ -0,0 +1,29 @@ +# Playbook to upgrade Docker to the max allowable version for an OpenShift cluster. +# +# Currently only supports upgrading 1.9.x to >= 1.10.x. +- hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + - add_host: + name: "{{ item }}" + groups: l_oo_all_hosts + with_items: g_all_hosts | default([]) + changed_when: false + +- hosts: l_oo_all_hosts + gather_facts: no + tasks: + - include_vars: ../../cluster_hosts.yml + +- include: ../../../../common/openshift-cluster/evaluate_groups.yml + vars: + # Do not allow adding hosts during upgrade. + g_new_master_hosts: [] + g_new_node_hosts: [] + openshift_cluster_id: "{{ cluster_id | default('default') }}" + openshift_deployment_type: "{{ deployment_type }}" + +- include: docker_upgrade.yml diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml deleted file mode 100644 index 9ade795f2..000000000 --- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/docker_upgrade.yml +++ /dev/null @@ -1,15 +0,0 @@ -- name: Check if Docker is installed - command: rpm -q docker - register: pkg_check - failed_when: pkg_check.rc > 1 - changed_when: no - -- name: Upgrade Docker - command: "{{ ansible_pkg_mgr}} update -y docker" - when: pkg_check.rc == 0 and g_docker_version.curr_version | version_compare('1.9','<') - register: docker_upgrade - -- name: Restart Docker - command: systemctl restart docker - when: docker_upgrade | changed - diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml index c93bf2a17..156e80c0f 100644 --- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml +++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml @@ -3,13 +3,13 @@ # The restart playbook should be run after this playbook completes. ############################################################################### -- name: Upgrade docker +- include: ../../../../byo/openshift-cluster/upgrades/docker/docker_upgrade.yml + +- name: Update Docker facts hosts: oo_masters_to_config:oo_nodes_to_config:oo_etcd_to_config roles: - openshift_facts tasks: - - include: docker_upgrade.yml - when: not openshift.common.is_atomic | bool - name: Set post docker install facts openshift_facts: role: "{{ item.role }}" diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml index 5f94500a0..3368e7cd9 100644 --- a/roles/docker/tasks/main.yml +++ b/roles/docker/tasks/main.yml @@ -1,41 +1,42 @@ --- -# tasks file for docker - -- name: Get current installed version if docker_version is specified +# Going forward we require Docker 1.10 or greater. If the user has a lesser version installed they must run a separate upgrade process. +- name: Get current installed Docker version command: "{{ repoquery_cmd }} --installed --qf '%{version}' docker" - when: not openshift.common.is_atomic | bool and docker_version != '' - register: docker_version_result + when: not openshift.common.is_atomic | bool + register: curr_docker_version changed_when: false -- stat: path=/etc/sysconfig/docker-storage - register: docker_storage_check +# TODO: The use of upgrading var will be removed in the coming upgrade refactor. This is a temporary +# fix to wory around the fact that right now, this role is called during upgrade, before we're +# ready to upgrade Docker. +- name: Fail if Docker upgrade is required + fail: + msg: "Docker {{ curr_docker_version.stdout }} must be upgraded to Docker 1.10 or greater" + when: not upgrading | bool and not curr_docker_version | skipped and curr_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<') -- name: Remove deferred deletion for downgrades from 1.9 +- name: Get latest available version of Docker command: > - sed -i 's/--storage-opt dm.use_deferred_deletion=true//' /etc/sysconfig/docker-storage - when: docker_storage_check.stat.exists | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<') - -- name: Downgrade docker if necessary - command: "{{ ansible_pkg_mgr }} swap -y docker-* docker-*{{ docker_version }}" - register: docker_downgrade_result - when: not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'gt') + {{ repoquery_cmd }} --qf '%{version}' "docker" + register: avail_docker_version + failed_when: false + changed_when: false + when: not curr_docker_version.stdout | default('0.0', True) | version_compare('1.10', '>=') and not openshift.common.is_atomic | bool -- name: Install docker - action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version if docker_version is defined and docker_version != '' else '' }} state=present" - when: not openshift.common.is_atomic | bool and docker_downgrade_result | skipped +- name: Verify Docker >= 1.10 is available + fail: + msg: "Docker {{ avail_docker_version.stdout }} is available, but 1.10 or greater is required" + when: not avail_docker_version | skipped and avail_docker_version.stdout | default('0.0', True) | version_compare('1.10', '<') -# If docker were enabled and started before we downgraded it may have entered a -# failed state. Check for that and clear it if necessary. -- name: Check that docker hasn't entered failed state - command: systemctl show docker - register: docker_state - changed_when: False +- stat: path=/etc/sysconfig/docker-storage + register: docker_storage_check -- name: Reset docker service state - command: systemctl reset-failed docker.service - when: " 'ActiveState=failed' in docker_state.stdout " +# Make sure Docker is installed, but does not update a running version. +# Docker upgrades are handled by a separate playbook. +- name: Install Docker + action: "{{ ansible_pkg_mgr }} name=docker state=present" + when: not openshift.common.is_atomic | bool -- name: Start the docker service +- name: Start the Docker service service: name: docker enabled: yes @@ -87,7 +88,7 @@ - restart docker when: "{{ 'http_proxy' in openshift.common or 'https_proxy' in openshift.common and docker_check.stat.isreg }}" -- name: Set various docker options +- name: Set various Docker options lineinfile: dest: /etc/sysconfig/docker regexp: '^OPTIONS=.*$' diff --git a/roles/openshift_cli/tasks/main.yml b/roles/openshift_cli/tasks/main.yml index bfa60e5b0..c0a712513 100644 --- a/roles/openshift_cli/tasks/main.yml +++ b/roles/openshift_cli/tasks/main.yml @@ -5,7 +5,7 @@ - name: Pull CLI Image command: > - docker pull {{ openshift.common.cli_image }}:{{ openshift_version }} + docker pull {{ openshift.common.cli_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }} when: openshift.common.is_containerized | bool - name: Create /usr/local/bin/openshift cli wrapper diff --git a/roles/openshift_cli/templates/openshift.j2 b/roles/openshift_cli/templates/openshift.j2 index 437e08aab..8a3f3a257 100644 --- a/roles/openshift_cli/templates/openshift.j2 +++ b/roles/openshift_cli/templates/openshift.j2 @@ -5,14 +5,14 @@ fi cmd=`basename $0` user=`id -u` group=`id -g` -image_tag={{ openshift_version }} +image_tag="{{ openshift_version }}" >&2 echo """ ================================================================================ ATTENTION: You are running ${cmd} via a wrapper around 'docker run {{ openshift.common.cli_image }}:${image_tag}'. This wrapper is intended only to be used to bootstrap an environment. Please install client tools on another host once you have granted cluster-admin -privileges to a user. +privileges to a user. {% if openshift.common.deployment_type in ['openshift-enterprise','atomic-enterprise'] %} See https://docs.openshift.com/enterprise/latest/cli_reference/get_started_cli.html {% else %} @@ -21,4 +21,8 @@ See https://docs.openshift.org/latest/cli_reference/get_started_cli.html ================================================================================= """ -docker run -i --privileged --net=host --user=${user}:${group} -v ~/.kube:/root/.kube -v /tmp:/tmp -v {{ openshift.common.config_base}}:{{ openshift.common.config_base }} -e KUBECONFIG=/root/.kube/config --entrypoint ${cmd} --rm {{ openshift.common.cli_image }}:${image_tag} "${@}" +if [ -n "$image_tag" ]; then + image_tag=":$image_tag" +fi + +docker run -i --privileged --net=host --user=${user}:${group} -v ~/.kube:/root/.kube -v /tmp:/tmp -v {{ openshift.common.config_base}}:{{ openshift.common.config_base }} -e KUBECONFIG=/root/.kube/config --entrypoint ${cmd} --rm {{ openshift.common.cli_image }}${image_tag} "${@}" diff --git a/roles/openshift_docker_facts/tasks/main.yml b/roles/openshift_docker_facts/tasks/main.yml index 43359dcb5..cdea90413 100644 --- a/roles/openshift_docker_facts/tasks/main.yml +++ b/roles/openshift_docker_facts/tasks/main.yml @@ -57,14 +57,3 @@ l_common_version: "{{ common_version.stdout | default('0.0', True) }}" when: not openshift.common.is_containerized | bool -- name: Set docker version to be installed - set_fact: - docker_version: "{{ '1.8.2' }}" - when: " ( l_common_version | version_compare('3.2','<') and openshift.common.service_type in ['openshift', 'atomic-openshift'] ) or - ( l_common_version | version_compare('1.1.4','<') and openshift.common.service_type == 'origin' )" - -- name: Set docker version to be installed - set_fact: - docker_version: "{{ '1.9.1' }}" - when: " ( l_common_version | version_compare('3.2','>') and openshift.common.service_type == 'atomic-openshift' ) or - ( l_common_version | version_compare('1.2','>') and openshift.common.service_type == 'origin' )" diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 52ea8bccf..28faee155 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -29,7 +29,7 @@ - name: Pull master image command: > - docker pull {{ openshift.master.master_image }}:{{ openshift_version }} + docker pull {{ openshift.master.master_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }} when: openshift.common.is_containerized | bool - name: Create openshift.common.data_dir diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index e8bd13855..657e99e87 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -39,12 +39,12 @@ - name: Pull node image command: > - docker pull {{ openshift.node.node_image }}:{{ openshift_version }} + docker pull {{ openshift.node.node_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }} when: openshift.common.is_containerized | bool - name: Pull OpenVSwitch image command: > - docker pull {{ openshift.node.ovs_image }}:{{ openshift_version }} + docker pull {{ openshift.node.ovs_image }}{{ ':' + openshift_version if openshift_version is defined and openshift_version != '' else '' }} when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool - name: Install the systemd units |