From 3cc2d6b019fa8a05ba480a7425cd654aa051355f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 6 Oct 2016 14:39:18 +0200 Subject: Check that OpenStack hostnames are resolvable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If they’re not, then use IP instead --- roles/openshift_facts/library/openshift_facts.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 9ffd399bc..0ee018c5c 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -364,12 +364,15 @@ def normalize_openstack_facts(metadata, facts): facts['network']['ip'] = local_ipv4 facts['network']['public_ip'] = metadata['ec2_compat']['public-ipv4'] - # TODO: verify local hostname makes sense and is resolvable - facts['network']['hostname'] = metadata['hostname'] - - # TODO: verify that public hostname makes sense and is resolvable - pub_h = metadata['ec2_compat']['public-hostname'] - facts['network']['public_hostname'] = pub_h + for f_var, h_var, ip_var in [('hostname', 'hostname', 'local-ipv4'), + ('public_hostname', 'public-hostname', 'public-ipv4')]: + try: + if socket.gethostbyname(metadata['ec2_compat'][h_var]) == metadata['ec2_compat'][ip_var]: + facts['network'][f_var] = metadata['ec2_compat'][h_var] + else: + facts['network'][f_var] = metadata['ec2_compat'][ip_var] + except socket.gaierror: + facts['network'][f_var] = metadata['ec2_compat'][ip_var] return facts -- cgit v1.2.3 From 1bc6d4390661fe18bebbc020b2c7b25972e80b41 Mon Sep 17 00:00:00 2001 From: Dan Mace Date: Fri, 7 Oct 2016 16:28:17 -0400 Subject: Retry failed master startup once Master startup can fail when ec2 transparently reallocates the block storage, causing etcd writes to temporarily fail. Retry failures blindly just once to allow time for this transient condition to to resolve and for systemd to restart the master (which will eventually succeed). https://github.com/coreos/etcd/issues/3864 https://github.com/openshift/origin/issues/6065 https://github.com/openshift/origin/issues/6447 --- roles/openshift_master/tasks/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index ce2f96723..645871ab4 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -168,10 +168,21 @@ - include: set_loopback_context.yml when: openshift.common.version_gte_3_2_or_1_2 +# TODO: Master startup can fail when ec2 transparently reallocates the block +# storage, causing etcd writes to temporarily fail. Retry failures blindly just +# once to allow time for this transient condition to to resolve and for systemd +# to restart the master (which will eventually succeed). +# +# https://github.com/coreos/etcd/issues/3864 +# https://github.com/openshift/origin/issues/6065 +# https://github.com/openshift/origin/issues/6447 - name: Start and enable master service: name={{ openshift.common.service_type }}-master enabled=yes state=started when: not openshift_master_ha | bool register: start_result + until: not start_result | failed + retries: 1 + delay: 60 notify: Verify API Server - name: Check for non-HA master service presence -- cgit v1.2.3 From 44d0816ecd1e55b98e4e76361072667c2ffdae40 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Fri, 7 Oct 2016 16:12:59 -0400 Subject: Improve how we handle containerized node failure on first startup --- roles/openshift_node/tasks/main.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index be07bd2d3..8e9c9f511 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -152,17 +152,9 @@ - name: Start and enable node service: name={{ openshift.common.service_type }}-node enabled=yes state=started register: node_start_result - ignore_errors: yes - -- name: Wait 30 seconds for docker initialization whenever node has failed - pause: - seconds: 30 - when: node_start_result | failed - -- name: Start and enable node again - service: name={{ openshift.common.service_type }}-node enabled=yes state=started - register: node_start_result - when: node_start_result | failed + until: not node_start_result | failed + retries: 1 + delay: 30 - set_fact: node_service_status_changed: "{{ node_start_result | changed }}" -- cgit v1.2.3 From ebef73fe94a0329011ca35ba33272789826ce282 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Mon, 10 Oct 2016 09:43:15 -0400 Subject: Apply same pattern to HA master services --- roles/openshift_master/tasks/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml index 645871ab4..1a59717c7 100644 --- a/roles/openshift_master/tasks/main.yml +++ b/roles/openshift_master/tasks/main.yml @@ -213,6 +213,9 @@ state: started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname == openshift_master_hosts[0] register: start_result + until: not start_result | failed + retries: 1 + delay: 60 - set_fact: master_api_service_status_changed: "{{ start_result | changed }}" @@ -229,6 +232,9 @@ state: started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname != openshift_master_hosts[0] register: start_result + until: not start_result | failed + retries: 1 + delay: 60 - set_fact: master_api_service_status_changed: "{{ start_result | changed }}" @@ -262,6 +268,9 @@ state: started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname == openshift_master_hosts[0] register: start_result + until: not start_result | failed + retries: 1 + delay: 60 - pause: seconds: 15 @@ -274,6 +283,9 @@ state: started when: openshift_master_ha | bool and openshift.master.cluster_method == 'native' and inventory_hostname != openshift_master_hosts[0] register: start_result + until: not start_result | failed + retries: 1 + delay: 60 - set_fact: master_controllers_service_status_changed: "{{ start_result | changed }}" -- cgit v1.2.3 From c5e95907b6311935a5fa7e56b2f00565b32e4fb6 Mon Sep 17 00:00:00 2001 From: Troy Dawson Date: Mon, 10 Oct 2016 08:43:59 -0500 Subject: Automatic commit of package [openshift-ansible] release [3.4.6-1]. --- .tito/packages/openshift-ansible | 2 +- openshift-ansible.spec | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.tito/packages/openshift-ansible b/.tito/packages/openshift-ansible index da02d3f12..4daa2cb96 100644 --- a/.tito/packages/openshift-ansible +++ b/.tito/packages/openshift-ansible @@ -1 +1 @@ -3.4.5-1 ./ +3.4.6-1 ./ diff --git a/openshift-ansible.spec b/openshift-ansible.spec index f4e89f906..5aaa4c2bc 100644 --- a/openshift-ansible.spec +++ b/openshift-ansible.spec @@ -5,7 +5,7 @@ } Name: openshift-ansible -Version: 3.4.5 +Version: 3.4.6 Release: 1%{?dist} Summary: Openshift and Atomic Enterprise Ansible License: ASL 2.0 @@ -249,6 +249,12 @@ Atomic OpenShift Utilities includes %changelog +* Mon Oct 10 2016 Troy Dawson 3.4.6-1 +- Retry failed master startup once (ironcladlou@gmail.com) +- [logging] Fix openshift_hosted_logging_fluentd_nodeselector + (sdodson@redhat.com) +- Changes for etcd servers (vishal.patil@nuagenetworks.net) + * Fri Oct 07 2016 Scott Dodson 3.4.5-1 - [a-o-i] -v disables quiet ansible config. (abutcher@redhat.com) -- cgit v1.2.3 From 492a513a253eb7a61b47f312f1002a62c7a7bfb0 Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Mon, 10 Oct 2016 11:52:41 -0400 Subject: Fix config and namespace for registry volume detection --- .../tasks/registry/storage/persistent_volume.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/roles/openshift_hosted/tasks/registry/storage/persistent_volume.yml b/roles/openshift_hosted/tasks/registry/storage/persistent_volume.yml index 60eefd71a..0172f5ca0 100644 --- a/roles/openshift_hosted/tasks/registry/storage/persistent_volume.yml +++ b/roles/openshift_hosted/tasks/registry/storage/persistent_volume.yml @@ -3,7 +3,11 @@ registry_volume_claim: "{{ openshift.hosted.registry.storage.volume.name }}-claim" - name: Determine if volume is already attached to dc/docker-registry - command: "{{ openshift.common.client_binary }} get -o template dc/docker-registry --template=\\{\\{.spec.template.spec.volumes\\}\\} --output-version=v1" + command: > + {{ openshift.common.client_binary }} + --config={{ openshift_hosted_kubeconfig }} + --namespace={{ openshift.hosted.registry.namespace | default('default') }} + get -o template dc/docker-registry --template=\\{\\{.spec.template.spec.volumes\\}\\} --output-version=v1 changed_when: false failed_when: false register: registry_volumes_output @@ -13,7 +17,10 @@ - name: Add volume to dc/docker-registry command: > - {{ openshift.common.client_binary }} volume dc/docker-registry + {{ openshift.common.client_binary }} + --config={{ openshift_hosted_kubeconfig }} + --namespace={{ openshift.hosted.registry.namespace | default('default') }} + volume dc/docker-registry --add --overwrite -t persistentVolumeClaim --claim-name={{ registry_volume_claim }} --name=registry-storage when: not volume_attached | bool -- cgit v1.2.3 From 0a8c565b88cb93dd2e9f8788ffa09d07764ab77d Mon Sep 17 00:00:00 2001 From: Scott Dodson Date: Mon, 10 Oct 2016 15:57:31 -0400 Subject: Set HTTPS_PROXY in example builddefaults_json --- inventory/byo/hosts.origin.example | 2 +- inventory/byo/hosts.ose.example | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inventory/byo/hosts.origin.example b/inventory/byo/hosts.origin.example index 706dd8080..0f0e223ce 100644 --- a/inventory/byo/hosts.origin.example +++ b/inventory/byo/hosts.origin.example @@ -599,7 +599,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', #openshift_builddefaults_git_http_proxy=http://USER:PASSWORD@HOST:PORT #openshift_builddefaults_git_https_proxy=https://USER:PASSWORD@HOST:PORT # Or you may optionally define your own serialized as json -#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"kind":"BuildDefaultsConfig","apiVersion":"v1","gitHTTPSProxy":"http://proxy.example.com.redhat.com:3128","gitHTTPProxy":"http://proxy.example.com.redhat.com:3128","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}]}}' +#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"kind":"BuildDefaultsConfig","apiVersion":"v1","gitHTTPSProxy":"http://proxy.example.com.redhat.com:3128","gitHTTPProxy":"http://proxy.example.com.redhat.com:3128","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"HTTPS_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}]}}' # masterConfig.volumeConfig.dynamicProvisioningEnabled, configurable as of 1.2/3.2, enabled by default #openshift_master_dynamic_provisioning_enabled=False diff --git a/inventory/byo/hosts.ose.example b/inventory/byo/hosts.ose.example index 09f1350b1..2e5b7564d 100644 --- a/inventory/byo/hosts.ose.example +++ b/inventory/byo/hosts.ose.example @@ -599,7 +599,7 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', #openshift_builddefaults_git_http_proxy=http://USER:PASSWORD@HOST:PORT #openshift_builddefaults_git_https_proxy=https://USER:PASSWORD@HOST:PORT # Or you may optionally define your own serialized as json -#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"kind":"BuildDefaultsConfig","apiVersion":"v1","gitHTTPSProxy":"http://proxy.example.com.redhat.com:3128","gitHTTPProxy":"http://proxy.example.com.redhat.com:3128","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}]}}' +#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"kind":"BuildDefaultsConfig","apiVersion":"v1","gitHTTPSProxy":"http://proxy.example.com.redhat.com:3128","gitHTTPProxy":"http://proxy.example.com.redhat.com:3128","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"HTTPS_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}]}}' # masterConfig.volumeConfig.dynamicProvisioningEnabled, configurable as of 1.2/3.2, enabled by default #openshift_master_dynamic_provisioning_enabled=False -- cgit v1.2.3