From 4a38bc1bbcb743d94a481c539ccd723efe436da0 Mon Sep 17 00:00:00 2001 From: Steve Milner Date: Thu, 6 Jul 2017 10:33:27 -0400 Subject: Test docker restart with retries 3 delay 30 --- roles/openshift_node_upgrade/tasks/restart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'roles/openshift_node_upgrade') diff --git a/roles/openshift_node_upgrade/tasks/restart.yml b/roles/openshift_node_upgrade/tasks/restart.yml index 6947223af..f228b6e08 100644 --- a/roles/openshift_node_upgrade/tasks/restart.yml +++ b/roles/openshift_node_upgrade/tasks/restart.yml @@ -19,7 +19,7 @@ state: started register: docker_start_result until: not docker_start_result | failed - retries: 1 + retries: 3 delay: 30 - name: Update docker facts -- cgit v1.2.3 From 36a7c5bf5c69511020c516a33ca6b3e57aff485d Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Thu, 6 Jul 2017 11:28:26 -0400 Subject: Add retries to node restart handlers --- roles/openshift_node_upgrade/handlers/main.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'roles/openshift_node_upgrade') diff --git a/roles/openshift_node_upgrade/handlers/main.yml b/roles/openshift_node_upgrade/handlers/main.yml index cb51416d4..c636f6fa3 100644 --- a/roles/openshift_node_upgrade/handlers/main.yml +++ b/roles/openshift_node_upgrade/handlers/main.yml @@ -10,5 +10,13 @@ when: (not skip_node_svc_handlers | default(False) | bool) and openshift.common.is_containerized | bool - name: restart node - systemd: name={{ openshift.common.service_type }}-node state=restarted - when: (not skip_node_svc_handlers | default(False) | bool) and not (node_service_status_changed | default(false) | bool) + systemd: + name: "{{ openshift.common.service_type }}-node" + state: restarted + register: l_openshift_node_upgrade_restart_node_result + until: not l_openshift_node_upgrade_restart_node_result | failed + retries: 3 + delay: 30 + when: + - (not skip_node_svc_handlers | default(False) | bool) + - not (node_service_status_changed | default(false) | bool) -- cgit v1.2.3 From b98dd4f1ac8582dbdee70128151f7e14c68c9b74 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Thu, 6 Jul 2017 12:13:38 -0400 Subject: Wrap docker stop in retries --- roles/openshift_node_upgrade/tasks/docker/upgrade.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'roles/openshift_node_upgrade') diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml index 416cf605a..ebe87d6fd 100644 --- a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml +++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml @@ -26,7 +26,13 @@ - debug: var=docker_image_count.stdout when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool -- service: name=docker state=stopped +- service: + name: docker + state: stopped + register: l_openshift_node_upgrade_docker_stop_result + until: not l_openshift_node_upgrade_docker_stop_result | failed + retries: 3 + delay: 30 - name: Upgrade Docker package: name=docker{{ '-' + docker_version }} state=present -- cgit v1.2.3 From a05fbeb6135864fedfb648644b06702ee1afea68 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Mon, 10 Jul 2017 13:47:51 -0400 Subject: Wrap additional service changes in retries --- roles/openshift_node_upgrade/handlers/main.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'roles/openshift_node_upgrade') diff --git a/roles/openshift_node_upgrade/handlers/main.yml b/roles/openshift_node_upgrade/handlers/main.yml index c636f6fa3..110dfe5ce 100644 --- a/roles/openshift_node_upgrade/handlers/main.yml +++ b/roles/openshift_node_upgrade/handlers/main.yml @@ -1,7 +1,13 @@ --- - name: restart openvswitch - systemd: name=openvswitch state=restarted + systemd: + name: openvswitch + state: restarted when: (not skip_node_svc_handlers | default(False) | bool) and not (ovs_service_status_changed | default(false) | bool) and openshift.common.use_openshift_sdn | bool + register: l_openshift_node_upgrade_stop_openvswitch_result + until: not l_openshift_node_upgrade_stop_openvswitch_result | failed + retries: 3 + delay: 30 notify: - restart openvswitch pause -- cgit v1.2.3 From 5a94fe5b074d01a3b16db8a05c47c31e484e5ebe Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Fri, 14 Jul 2017 12:13:25 -0400 Subject: Add drain retries after 60 second delay --- roles/openshift_node_upgrade/README.md | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'roles/openshift_node_upgrade') diff --git a/roles/openshift_node_upgrade/README.md b/roles/openshift_node_upgrade/README.md index 8b388cc6a..4e6229bfb 100644 --- a/roles/openshift_node_upgrade/README.md +++ b/roles/openshift_node_upgrade/README.md @@ -84,6 +84,11 @@ Including an example of how to use your role (for instance, with variables passe command: > {{ hostvars[groups.oo_first_master.0].openshift.common.admin_binary }} drain {{ openshift.node.nodename | lower }} --force --delete-local-data --ignore-daemonsets delegate_to: "{{ groups.oo_first_master.0 }}" + register: l_docker_upgrade_drain_result + until: not l_docker_upgrade_drain_result | failed + retries: 60 + delay: 60 + roles: - openshift_facts -- cgit v1.2.3