Merge pull request #1718 from brenton/docker1

Containerized installs on RHEL were downgrading docker unnecessarily

17 files changed, 93 insertions, 67 deletions

diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py
index 127bd69cf..f6cc2edde 100644
--- a/filter_plugins/oo_filters.py
+++ b/filter_plugins/oo_filters.py
@@ -791,7 +791,7 @@ class FilterModule(object):
         return retval
 
     @staticmethod
-    def oo_image_tag_to_rpm_version(version):
+    def oo_image_tag_to_rpm_version(version, include_dash=False):
         """ Convert an image tag string to an RPM version if necessary
             Empty strings and strings that are already in rpm version format
             are ignored.
@@ -802,7 +802,10 @@ class FilterModule(object):
             raise errors.AnsibleFilterError("|failed expects a string or unicode")
 
         if version.startswith("v"):
-            version = "-" + version.replace("v", "")
+            version = version.replace("v", "")
+
+            if include_dash:
+                version = "-" + version
 
         return version
 
diff --git a/playbooks/common/openshift-cluster/upgrades/files/openshift_container_versions.sh b/playbooks/common/openshift-cluster/upgrades/files/openshift_container_versions.sh
index 7a1edf38f..96944a78b 100644
--- a/playbooks/common/openshift-cluster/upgrades/files/openshift_container_versions.sh
+++ b/playbooks/common/openshift-cluster/upgrades/files/openshift_container_versions.sh
@@ -3,19 +3,19 @@
 # Here we don't really care if this is a master, api, controller or node image.
 # We just need to know the version of one of them.
 unit_file=$(ls /etc/systemd/system/${1}*.service | head -n1)
-installed_container_name=$(basename -s .service ${unit_file})
-installed=$(docker exec ${installed_container_name} openshift version 2> /dev/null | grep openshift | awk '{ print $2 }' | cut -f1 -d"-" | tr -d 'v')
 
 if [ ${1} == "origin" ]; then
     image_name="openshift/origin"
 elif grep aep $unit_file 2>&1 > /dev/null; then
-    image_name="aep3/aep"
+    image_name="aep3/node"
 elif grep openshift3 $unit_file 2>&1 > /dev/null; then
-    image_name="openshift3/ose"
+    image_name="openshift3/node"
 fi
 
+installed=$(docker run --rm --entrypoint=/bin/openshift ${image_name} version 2> /dev/null | grep openshift | awk '{ print $2 }' | cut -f1 -d"-" | tr -d 'v')
+
 docker pull ${image_name} 2>&1 > /dev/null
-available=$(docker run --rm ${image_name} version 2> /dev/null | grep openshift | awk '{ print $2 }' | cut -f1 -d"-" | tr -d 'v')
+available=$(docker run --rm --entrypoint=/bin/openshift ${image_name} version 2> /dev/null | grep openshift | awk '{ print $2 }' | cut -f1 -d"-" | tr -d 'v')
 
 echo "---"
 echo "curr_version: ${installed}"
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/pre.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/pre.yml
index 6e6f31195..21480ba55 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/pre.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/pre.yml
@@ -38,7 +38,7 @@
       msg: >
         This upgrade does not support Pacemaker:
         https://docs.openshift.{{ g_pacemaker_upgrade_url_segment }}/install_config/upgrading/pacemaker_to_native_ha.html
-    when: openshift.master.cluster_method == 'pacemaker'
+    when: openshift.master.cluster_method is defined and openshift.master.cluster_method == 'pacemaker'
 
   - fail:
       msg: >
@@ -57,6 +57,11 @@
   roles:
   - openshift_facts
   tasks:
+  - openshift_facts:
+      role: master
+      local_facts:
+        ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+
   - name: Ensure Master is running
     service:
       name: "{{ openshift.common.service_type }}-master"
@@ -77,11 +82,6 @@
       state: started
       enabled: yes
     when: openshift.master.ha is defined and openshift.master.ha | bool and openshift.common.is_containerized | bool
-  post_tasks:
-  - openshift_facts:
-      role: master
-      local_facts:
-        ha: "{{ groups.oo_masters_to_config | length > 1 }}"
 
 - name: Verify upgrade can proceed
   hosts: oo_nodes_to_config
@@ -105,8 +105,12 @@
   - include: ../../../../../roles/openshift_master/handlers/main.yml
   - include: ../../../../../roles/openshift_node/handlers/main.yml
   roles:
+  # We want the cli role to evaluate so that the containerized oc/oadm wrappers
+  # are modified to use the correct image tag.  However, this can trigger a
+  # docker restart if new configuration is laid down which would immediately
+  # pull the latest image and defeat the purpose of these tasks.
   - openshift_cli
-  tasks:
+  pre_tasks:
   - name: Clean package cache
     command: "{{ ansible_pkg_mgr }} clean all"
     when: not openshift.common.is_atomic | bool
@@ -147,6 +151,10 @@
 
   - fail:
       msg: Verifying the correct version was found
+    when: g_aos_versions.curr_version == ""
+
+  - fail:
+      msg: Verifying the correct version was found
     when: verify_upgrade_version is defined and g_new_version != verify_upgrade_version
 
   - include_vars: ../../../../../roles/openshift_master/vars/main.yml
diff --git a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
index 041ad5a9a..eb12fcabe 100644
--- a/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/v3_1_to_v3_2/upgrade.yml
@@ -1,33 +1,4 @@
 ---
-# This is a workaround for authenticated registries
-- name: Download new images
-  hosts: oo_nodes_to_config
-  roles:
-  - openshift_facts
-  tasks:
-  - name: Pull Images
-    command: >
-      docker pull {{ item }}:v{{ g_new_version }}
-    with_items:
-    - "{{ openshift.node.node_image }}"
-    - "{{ openshift.node.ovs_image }}"
-    - "{{ openshift.common.pod_image }}"
-    - "{{ openshift.common.router_image }}"
-    - "{{ openshift.common.registry_image }}"
-    - "{{ openshift.common.deployer_image }}"
-
-# This is a workaround for authenticated registries
-- name: Download new images
-  hosts: oo_masters_to_config
-  roles:
-  - openshift_facts
-  tasks:
-  - name: Pull Images
-    command: >
-      docker pull {{ item }}:v{{ g_new_version }}
-    with_items:
-    - "{{ openshift.master.master_image }}"
-
 ###############################################################################
 # The restart playbook should be run after this playbook completes.
 ###############################################################################
@@ -40,6 +11,20 @@
   - include: docker_upgrade.yml
     when: not openshift.common.is_atomic | bool
 
+# The cli image is used by openshift_facts to determine the currently installed
+# version.  We need to explicitly pull the latest image to handle cases where
+# the locally cached 'latest' tag is older the g_new_version.
+- name: Download cli image
+  hosts: oo_masters_to_config:oo_nodes_to_config
+  roles:
+  - openshift_facts
+  tasks:
+  - name: Pull Images
+    command: >
+      docker pull {{ item }}:latest
+    with_items:
+    - "{{ openshift.common.cli_image }}"
+
 ###############################################################################
 # Upgrade Masters
 ###############################################################################
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 1d8493938..89648b321 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -7,22 +7,27 @@
   register: docker_version_result
   changed_when: false
 
+- stat: path=/etc/sysconfig/docker-storage
+  register: docker_storage_check
+
+- name: Remove deferred deletion for downgrades from 1.9
+  command: >
+    sed -i 's/--storage-opt dm.use_deferred_deletion=true//' /etc/sysconfig/docker-storage
+  when: docker_storage_check.stat.exists | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<')
+
 - name: Downgrade docker if necessary
   command: "{{ ansible_pkg_mgr }} downgrade -y docker-{{ docker_version }}"
   register: docker_downgrade_result
   when: not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'gt')
 
 - name: Install docker
-  action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version if docker_version != '' else '' }} state=present"
-  when: not openshift.common.is_atomic | bool and not docker_downgrade_result | changed
+  action: "{{ ansible_pkg_mgr }} name=docker{{ '-' + docker_version if docker_version is defined and docker_version != '' else '' }} state=present"
+  when: not openshift.common.is_atomic | bool and not docker_version_result | skipped and docker_version_result.stdout | default('0.0', True) | version_compare(docker_version, 'lt')
 
-- stat: path=/etc/sysconfig/docker-storage
-  register: docker_storage_check
-
-- name: Remove deferred deletion for downgrades from 1.9
-  command: >
-    sed -i 's/--storage-opt dm.use_deferred_deletion=true//' /etc/sysconfig/docker-storage
-  when: docker_downgrade_result | changed and docker_storage_check.stat.exists | bool and docker_version_result.stdout | default('0.0', True) | version_compare('1.9', '>=') and docker_version | version_compare('1.9', '<')
+# We're getting ready to start docker.  This is a workaround for cases where it
+# seems a package install/upgrade/downgrade has rebooted docker and crashed it.
+- name: Reset docker service state
+  command: systemctl reset-failed docker.service
 
 - name: enable and start the docker service
   service:
diff --git a/roles/etcd/templates/etcd.docker.service b/roles/etcd/templates/etcd.docker.service
index b8dbefa64..cf957ede8 100644
--- a/roles/etcd/templates/etcd.docker.service
+++ b/roles/etcd/templates/etcd.docker.service
@@ -11,6 +11,7 @@ ExecStart=/usr/bin/docker run --name {{ etcd_service }} --rm -v /var/lib/etcd:/v
 ExecStop=/usr/bin/docker stop {{ etcd_service }}
 SyslogIdentifier=etcd_container
 Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service
diff --git a/roles/openshift_common/tasks/main.yml b/roles/openshift_common/tasks/main.yml
index 59ca659b5..eda43b9f8 100644
--- a/roles/openshift_common/tasks/main.yml
+++ b/roles/openshift_common/tasks/main.yml
@@ -33,7 +33,7 @@
 # versions or image tags.  openshift_common's usage requires that it be a RPM
 # version and openshift_cli expects it to be an image tag.
 - name: Install the base package for versioning
-  action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}{{ openshift_version | default('') | oo_image_tag_to_rpm_version }} state=present"
+  action: "{{ ansible_pkg_mgr }} name={{ openshift.common.service_type }}{{ openshift_version | default('') | oo_image_tag_to_rpm_version(include_dash=True) }} state=present"
   when: not openshift.common.is_containerized | bool
 
 - name: Set version facts
diff --git a/roles/openshift_docker_facts/tasks/main.yml b/roles/openshift_docker_facts/tasks/main.yml
index eb762e33f..7ea359af1 100644
--- a/roles/openshift_docker_facts/tasks/main.yml
+++ b/roles/openshift_docker_facts/tasks/main.yml
@@ -46,10 +46,24 @@
   register: common_version
   failed_when: false
   changed_when: false
-  when: not openshift.common.is_atomic | bool
+  when: not openshift.common.is_containerized | bool
+
+- set_fact:
+    l_common_version: "{{ openshift.common.image_tag | default('0.0', True) | oo_image_tag_to_rpm_version }}"
+  when: openshift.common.is_containerized | bool
+
+- set_fact:
+    l_common_version: "{{ common_version.stdout | default('0.0', True) }}"
+  when: not openshift.common.is_containerized | bool
 
 - name: Set docker version to be installed
   set_fact:
     docker_version: "{{ '1.8.2' }}"
-  when: " ( common_version.stdout | default('0.0', True) | version_compare('3.2','<') and openshift.common.service_type == 'atomic-openshift' ) or
-          ( common_version.stdout | default('0.0', True) | version_compare('1.1.4','<') and openshift.common.service_type == 'origin' )"
+  when: " ( l_common_version | version_compare('3.2','<') and openshift.common.service_type == 'atomic-openshift' ) or
+          ( l_common_version | version_compare('1.1.4','<') and openshift.common.service_type == 'origin' )"
+
+- name: Set docker version to be installed
+  set_fact:
+    docker_version: "{{ '1.9.1' }}"
+  when: " ( l_common_version | version_compare('3.2','>') and openshift.common.service_type == 'atomic-openshift' ) or
+          ( l_common_version | version_compare('1.2','>') and openshift.common.service_type == 'origin' )"
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 92d650550..2b68f27b7 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -1080,13 +1080,9 @@ def get_openshift_version(facts, cli_image=None):
         elif 'node' in facts:
             container = facts['common']['service_type'] + '-node'
 
-        if container is not None:
-            exit_code, output, _ = module.run_command(['docker', 'exec', container, 'openshift', 'version'])
-            # if for some reason the container is installed but not running
-            # we'll fall back to using docker run later in this method.
-            if exit_code == 0:
-                version = parse_openshift_version(output)
-
+	# Try to get the version fromthe available cli image _before_ resorting
+	# to exec'ing in to the running container.  This is to be more fault
+	# tolerant in environments where the container is not running.
         if version is None and cli_image is not None:
             # Assume we haven't installed the environment yet and we need
             # to query the latest image, but only if docker is installed
@@ -1094,6 +1090,13 @@ def get_openshift_version(facts, cli_image=None):
                 exit_code, output, _ = module.run_command(['docker', 'run', '--rm', cli_image, 'version'])
                 version = parse_openshift_version(output)
 
+        if version is None and container is not None:
+            exit_code, output, _ = module.run_command(['docker', 'exec', container, 'openshift', 'version'])
+            # if for some reason the container is installed but not running
+            # we'll fall back to using docker run later in this method.
+            if exit_code == 0:
+                version = parse_openshift_version(output)
+
     return version
 
 def parse_openshift_version(output):
@@ -1351,7 +1354,7 @@ def set_container_facts_if_unset(facts):
         facts['common']['admin_binary'] = '/usr/local/bin/oadm'
         facts['common']['client_binary'] = '/usr/local/bin/oc'
         openshift_version = get_openshift_version(facts, cli_image)
-        if openshift_version is not None:
+        if openshift_version is not None and openshift_version is not "":
             base_version = openshift_version.split('-')[0]
             facts['common']['image_tag'] = "v" + base_version
 
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index 70c7ef4e4..f7dfb11f7 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -1,17 +1,17 @@
 ---
 - name: restart master
   service: name={{ openshift.common.service_type }}-master state=restarted
-  when: (openshift.master.ha is defined and not openshift.master.ha | bool) and (not (master_service_status_changed | default(false) | bool))
+  when: (openshift.master.ha is not defined or not openshift.master.ha | bool) and (not (master_service_status_changed | default(false) | bool))
   notify: Verify API Server
 
 - name: restart master api
   service: name={{ openshift.common.service_type }}-master-api state=restarted
-  when: (openshift.master.ha is defined and openshift_master_ha | bool) and (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
+  when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
   notify: Verify API Server
 
 - name: restart master controllers
   service: name={{ openshift.common.service_type }}-master-controllers state=restarted
-  when: (openshift.master.ha is defined and openshift_master_ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
+  when: (openshift.master.ha is defined and openshift.master.ha | bool) and (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
 
 - name: Verify API Server
   # Using curl here since the uri module requires python-httplib2 and
diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2
index 66d76978e..df1dbb85e 100644
--- a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-api.service.j2
@@ -20,6 +20,7 @@ LimitCORE=infinity
 WorkingDirectory={{ openshift.common.data_dir }}
 SyslogIdentifier={{ openshift.common.service_type }}-master-api
 Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service
diff --git a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2
index 75759c133..5ff2edae4 100644
--- a/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2
+++ b/roles/openshift_master/templates/docker-cluster/atomic-openshift-master-controllers.service.j2
@@ -19,6 +19,7 @@ LimitCORE=infinity
 WorkingDirectory={{ openshift.common.data_dir }}
 SyslogIdentifier={{ openshift.common.service_type }}-master-controllers
 Restart=on-failure
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service
diff --git a/roles/openshift_master/templates/docker/master.docker.service.j2 b/roles/openshift_master/templates/docker/master.docker.service.j2
index d02fc5342..97f698b68 100644
--- a/roles/openshift_master/templates/docker/master.docker.service.j2
+++ b/roles/openshift_master/templates/docker/master.docker.service.j2
@@ -12,6 +12,7 @@ ExecStart=/usr/bin/docker run --rm --privileged --net=host --name {{ openshift.c
 ExecStartPost=/usr/bin/sleep 10
 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-master
 Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2
index e61418eb6..1f50fdce1 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.service.j2
@@ -15,6 +15,7 @@ LimitNOFILE=131072
 LimitCORE=infinity
 WorkingDirectory={{ openshift.common.data_dir }}
 SyslogIdentifier=atomic-openshift-master-api
+RestartSec=5s
 
 [Install]
 WantedBy=multi-user.target
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2
index ba4c1e0e8..bf62696f0 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.service.j2
@@ -20,6 +20,7 @@ LimitCORE=infinity
 WorkingDirectory={{ openshift.common.data_dir }}
 SyslogIdentifier={{ openshift.common.service_type }}-master-controllers
 Restart=on-failure
+RestartSec=5s
 
 [Install]
 WantedBy=multi-user.target
diff --git a/roles/openshift_node/templates/openshift.docker.node.service b/roles/openshift_node/templates/openshift.docker.node.service
index ff5a97fe0..30f09b250 100644
--- a/roles/openshift_node/templates/openshift.docker.node.service
+++ b/roles/openshift_node/templates/openshift.docker.node.service
@@ -17,6 +17,7 @@ ExecStartPost=/usr/bin/sleep 10
 ExecStop=/usr/bin/docker stop {{ openshift.common.service_type }}-node
 SyslogIdentifier={{ openshift.common.service_type }}-node
 Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service
diff --git a/roles/openshift_node/templates/openvswitch.docker.service b/roles/openshift_node/templates/openvswitch.docker.service
index 8052a3a39..1e1f8967d 100644
--- a/roles/openshift_node/templates/openvswitch.docker.service
+++ b/roles/openshift_node/templates/openvswitch.docker.service
@@ -11,6 +11,7 @@ ExecStartPost=/usr/bin/sleep 5
 ExecStop=/usr/bin/docker stop openvswitch
 SyslogIdentifier=openvswitch
 Restart=always
+RestartSec=5s
 
 [Install]
 WantedBy=docker.service