diff options
17 files changed, 191 insertions, 175 deletions
diff --git a/playbooks/common/openshift-checks/install.yml b/playbooks/common/openshift-checks/install.yml index 6701a2e15..93cf6c359 100644 --- a/playbooks/common/openshift-checks/install.yml +++ b/playbooks/common/openshift-checks/install.yml @@ -1,13 +1,15 @@ --- - name: Health Check Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Health Check 'In Progress' + run_once: true set_stats: data: - installer_phase_health: "In Progress" - aggregate: false + installer_phase_health: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: OpenShift Health Checks hosts: oo_all_hosts @@ -37,11 +39,13 @@ - docker_image_availability - name: Health Check Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Health Check 'Complete' + run_once: true set_stats: data: - installer_phase_health: "Complete" - aggregate: false + installer_phase_health: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/openshift_hosted.yml b/playbooks/common/openshift-cluster/openshift_hosted.yml index 281ccce2e..15ee60dc0 100644 --- a/playbooks/common/openshift-cluster/openshift_hosted.yml +++ b/playbooks/common/openshift-cluster/openshift_hosted.yml @@ -1,13 +1,15 @@ --- - name: Hosted Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Hosted install 'In Progress' + run_once: true set_stats: data: - installer_phase_hosted: "In Progress" - aggregate: false + installer_phase_hosted: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - include: create_persistent_volumes.yml @@ -30,11 +32,13 @@ - openshift_crio_enable_docker_gc | default(False) | bool - name: Hosted Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Hosted install 'Complete' + run_once: true set_stats: data: - installer_phase_hosted: "Complete" - aggregate: false + installer_phase_hosted: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/openshift_logging.yml b/playbooks/common/openshift-cluster/openshift_logging.yml index 529a4c939..bc59bd95a 100644 --- a/playbooks/common/openshift-cluster/openshift_logging.yml +++ b/playbooks/common/openshift-cluster/openshift_logging.yml @@ -1,13 +1,15 @@ --- - name: Logging Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Logging install 'In Progress' + run_once: true set_stats: data: - installer_phase_logging: "In Progress" - aggregate: false + installer_phase_logging: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: OpenShift Aggregated Logging hosts: oo_first_master @@ -23,11 +25,13 @@ tasks_from: update_master_config - name: Logging Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Logging install 'Complete' + run_once: true set_stats: data: - installer_phase_logging: "Complete" - aggregate: false + installer_phase_logging: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/openshift_metrics.yml b/playbooks/common/openshift-cluster/openshift_metrics.yml index 9c0bd489b..80cd93e5f 100644 --- a/playbooks/common/openshift-cluster/openshift_metrics.yml +++ b/playbooks/common/openshift-cluster/openshift_metrics.yml @@ -1,13 +1,15 @@ --- - name: Metrics Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Metrics install 'In Progress' + run_once: true set_stats: data: - installer_phase_metrics: "In Progress" - aggregate: false + installer_phase_metrics: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: OpenShift Metrics hosts: oo_first_master @@ -24,11 +26,13 @@ tasks_from: update_master_config.yaml - name: Metrics Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Metrics install 'Complete' + run_once: true set_stats: data: - installer_phase_metrics: "Complete" - aggregate: false + installer_phase_metrics: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/openshift_prometheus.yml b/playbooks/common/openshift-cluster/openshift_prometheus.yml index a73b294a5..7aa9a16e6 100644 --- a/playbooks/common/openshift-cluster/openshift_prometheus.yml +++ b/playbooks/common/openshift-cluster/openshift_prometheus.yml @@ -1,13 +1,15 @@ --- - name: Prometheus Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Prometheus install 'In Progress' + run_once: true set_stats: data: - installer_phase_prometheus: "In Progress" - aggregate: false + installer_phase_prometheus: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Create Hosted Resources - openshift_prometheus hosts: oo_first_master @@ -15,11 +17,13 @@ - role: openshift_prometheus - name: Prometheus Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Prometheus install 'Complete' + run_once: true set_stats: data: - installer_phase_prometheus: "Complete" - aggregate: false + installer_phase_prometheus: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/service_catalog.yml b/playbooks/common/openshift-cluster/service_catalog.yml index bd964b2ce..7bb8511f6 100644 --- a/playbooks/common/openshift-cluster/service_catalog.yml +++ b/playbooks/common/openshift-cluster/service_catalog.yml @@ -1,13 +1,15 @@ --- - name: Service Catalog Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Service Catalog install 'In Progress' + run_once: true set_stats: data: - installer_phase_servicecatalog: "In Progress" - aggregate: false + installer_phase_servicecatalog: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Service Catalog hosts: oo_first_master @@ -19,11 +21,13 @@ first_master: "{{ groups.oo_first_master[0] }}" - name: Service Catalog Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Service Catalog install 'Complete' + run_once: true set_stats: data: - installer_phase_servicecatalog: "Complete" - aggregate: false + installer_phase_servicecatalog: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-cluster/std_include.yml b/playbooks/common/openshift-cluster/std_include.yml index 45b34c8bd..fe376fe31 100644 --- a/playbooks/common/openshift-cluster/std_include.yml +++ b/playbooks/common/openshift-cluster/std_include.yml @@ -1,15 +1,17 @@ --- - name: Initialization Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false roles: - installer_checkpoint tasks: - name: Set install initialization 'In Progress' + run_once: true set_stats: data: - installer_phase_initialize: "In Progress" - aggregate: false + installer_phase_initialize: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - include: evaluate_groups.yml tags: @@ -36,11 +38,13 @@ - always - name: Initialization Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set install initialization 'Complete' + run_once: true set_stats: data: - installer_phase_initialize: "Complete" - aggregate: false + installer_phase_initialize: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml index 48d46bbb0..3fe483785 100644 --- a/playbooks/common/openshift-etcd/config.yml +++ b/playbooks/common/openshift-etcd/config.yml @@ -1,13 +1,15 @@ --- - name: etcd Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set etcd install 'In Progress' + run_once: true set_stats: data: - installer_phase_etcd: "In Progress" - aggregate: false + installer_phase_etcd: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - include: ca.yml @@ -26,11 +28,13 @@ - role: nickhammond.logrotate - name: etcd Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set etcd install 'Complete' + run_once: true set_stats: data: - installer_phase_etcd: "Complete" - aggregate: false + installer_phase_etcd: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-glusterfs/config.yml b/playbooks/common/openshift-glusterfs/config.yml index c2ae5f313..19e14ab3e 100644 --- a/playbooks/common/openshift-glusterfs/config.yml +++ b/playbooks/common/openshift-glusterfs/config.yml @@ -1,13 +1,15 @@ --- - name: GlusterFS Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set GlusterFS install 'In Progress' + run_once: true set_stats: data: - installer_phase_glusterfs: "In Progress" - aggregate: false + installer_phase_glusterfs: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Open firewall ports for GlusterFS nodes hosts: glusterfs @@ -46,11 +48,13 @@ when: groups.oo_glusterfs_to_config | default([]) | count > 0 - name: GlusterFS Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set GlusterFS install 'Complete' + run_once: true set_stats: data: - installer_phase_glusterfs: "Complete" - aggregate: false + installer_phase_glusterfs: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml index 2a703cb61..d737b836b 100644 --- a/playbooks/common/openshift-loadbalancer/config.yml +++ b/playbooks/common/openshift-loadbalancer/config.yml @@ -1,13 +1,15 @@ --- - name: Load Balancer Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set load balancer install 'In Progress' + run_once: true set_stats: data: - installer_phase_loadbalancer: "In Progress" - aggregate: false + installer_phase_loadbalancer: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Configure firewall and docker for load balancers hosts: oo_lb_to_config:!oo_masters_to_config:!oo_nodes_to_config @@ -37,11 +39,13 @@ - role: tuned - name: Load Balancer Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set load balancer install 'Complete' + run_once: true set_stats: data: - installer_phase_loadbalancer: "Complete" - aggregate: false + installer_phase_loadbalancer: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-management/config.yml b/playbooks/common/openshift-management/config.yml index 908679e81..3f1cdf713 100644 --- a/playbooks/common/openshift-management/config.yml +++ b/playbooks/common/openshift-management/config.yml @@ -1,13 +1,15 @@ --- - name: Management Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Management install 'In Progress' + run_once: true set_stats: data: - installer_phase_management: "In Progress" - aggregate: false + installer_phase_management: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Setup CFME hosts: oo_first_master @@ -25,11 +27,13 @@ template_dir: "{{ hostvars[groups.masters.0].r_openshift_management_mktemp.stdout }}" - name: Management Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Management install 'Complete' + run_once: true set_stats: data: - installer_phase_management: "Complete" - aggregate: false + installer_phase_management: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-master/additional_config.yml b/playbooks/common/openshift-master/additional_config.yml index 350557f19..4fef5b923 100644 --- a/playbooks/common/openshift-master/additional_config.yml +++ b/playbooks/common/openshift-master/additional_config.yml @@ -1,13 +1,15 @@ --- - name: Master Additional Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Master Additional install 'In Progress' + run_once: true set_stats: data: - installer_phase_master_additional: "In Progress" - aggregate: false + installer_phase_master_additional: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Additional master configuration hosts: oo_first_master @@ -36,11 +38,13 @@ when: openshift_use_flannel | default(false) | bool - name: Master Additional Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Master Additional install 'Complete' + run_once: true set_stats: data: - installer_phase_master_additional: "Complete" - aggregate: false + installer_phase_master_additional: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index 7ce0362ef..6b0fd6b7c 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -1,13 +1,15 @@ --- - name: Master Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Master install 'In Progress' + run_once: true set_stats: data: - installer_phase_master: "In Progress" - aggregate: false + installer_phase_master: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - include: certificates.yml @@ -238,11 +240,13 @@ r_openshift_excluder_service_type: "{{ openshift.common.service_type }}" - name: Master Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Master install 'Complete' + run_once: true set_stats: data: - installer_phase_master: "Complete" - aggregate: false + installer_phase_master: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-nfs/config.yml b/playbooks/common/openshift-nfs/config.yml index ce672daf5..6ea77e00b 100644 --- a/playbooks/common/openshift-nfs/config.yml +++ b/playbooks/common/openshift-nfs/config.yml @@ -1,13 +1,15 @@ --- - name: NFS Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set NFS install 'In Progress' + run_once: true set_stats: data: - installer_phase_nfs: "In Progress" - aggregate: false + installer_phase_nfs: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - name: Configure nfs hosts: oo_nfs_to_config @@ -16,11 +18,13 @@ - role: openshift_storage_nfs - name: NFS Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set NFS install 'Complete' + run_once: true set_stats: data: - installer_phase_nfs: "Complete" - aggregate: false + installer_phase_nfs: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml index 4f8f98aef..28e3c1b1b 100644 --- a/playbooks/common/openshift-node/config.yml +++ b/playbooks/common/openshift-node/config.yml @@ -1,13 +1,15 @@ --- - name: Node Install Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Node install 'In Progress' + run_once: true set_stats: data: - installer_phase_node: "In Progress" - aggregate: false + installer_phase_node: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" - include: certificates.yml @@ -24,11 +26,13 @@ - include: enable_excluders.yml - name: Node Install Checkpoint End - hosts: oo_all_hosts + hosts: all gather_facts: false tasks: - name: Set Node install 'Complete' + run_once: true set_stats: data: - installer_phase_node: "Complete" - aggregate: false + installer_phase_node: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" diff --git a/roles/installer_checkpoint/README.md b/roles/installer_checkpoint/README.md index 83e00e504..18d182ec9 100644 --- a/roles/installer_checkpoint/README.md +++ b/roles/installer_checkpoint/README.md @@ -92,31 +92,34 @@ phase/component and then a final play for setting `installer_hase_initialize` to # common/openshift-cluster/std_include.yml --- - name: Initialization Checkpoint Start - hosts: oo_all_hosts + hosts: all gather_facts: false roles: - installer_checkpoint tasks: - name: Set install initialization 'In Progress' + run_once: true set_stats: data: - installer_phase_initialize: "In Progress" - aggregate: false + installer_phase_initialize: + status: "In Progress" + start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" #... # Various plays here #... - name: Initialization Checkpoint End - hosts: localhost - connection: local + hosts: all gather_facts: false tasks: - name: Set install initialization 'Complete' + run_once: true set_stats: data: - installer_phase_initialize: "Complete" - aggregate: false + installer_phase_initialize: + status: "Complete" + end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}" ``` Each phase or component of the installer will follow a similar pattern, with the @@ -139,37 +142,25 @@ localhost : ok=24 changed=0 unreachable=0 failed=0 INSTALLER STATUS *************************************************************** -Initialization : Complete -etcd Install : Complete -NFS Install : Not Started -Load balancer Install : Not Started -Master Install : Complete -Master Additional Install : Complete -Node Install : Complete -GlusterFS Install : Not Started -Hosted Install : Complete -Metrics Install : Not Started -Logging Install : Not Started -Service Catalog Install : Not Started +Initialization : Complete (0:02:14) +Health Check : Complete (0:01:10) +etcd Install : Complete (0:02:01) +Master Install : Complete (0:11:43) +Master Additional Install : Complete (0:00:54) +Node Install : Complete (0:14:11) +Hosted Install : Complete (0:03:28) ``` Example display if a failure occurs during execution: ``` INSTALLER STATUS *************************************************************** -Initialization : Complete -etcd Install : Complete -NFS Install : Not Started -Load balancer Install : Not Started -Master Install : In Progress - This phase can be restarted by running: playbooks/byo/openshift-master/config.yml -Master Additional Install : Not Started -Node Install : Not Started -GlusterFS Install : Not Started -Hosted Install : Not Started -Metrics Install : Not Started -Logging Install : Not Started -Service Catalog Install : Not Started +Initialization : Complete (0:02:14) +Health Check : Complete (0:01:10) +etcd Install : Complete (0:02:58) +Master Install : Complete (0:09:20) +Master Additional Install : In Progress (0:20:04) + This phase can be restarted by running: playbooks/byo/openshift-master/additional_config.yml ``` [set_stats]: http://docs.ansible.com/ansible/latest/set_stats_module.html diff --git a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py index b17358882..556e9127f 100644 --- a/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py +++ b/roles/installer_checkpoint/callback_plugins/installer_checkpoint.py @@ -1,60 +1,10 @@ """Ansible callback plugin to print a summary completion status of installation phases. """ +from datetime import datetime from ansible.plugins.callback import CallbackBase from ansible import constants as C -DOCUMENTATION = ''' - -''' - -EXAMPLES = ''' ---------------------------------------------- -Example display of a successful playbook run: - -PLAY RECAP ********************************************************************* -master01.example.com : ok=158 changed=16 unreachable=0 failed=0 -node01.example.com : ok=469 changed=74 unreachable=0 failed=0 -node02.example.com : ok=157 changed=17 unreachable=0 failed=0 -localhost : ok=24 changed=0 unreachable=0 failed=0 - - -INSTALLER STATUS *************************************************************** -Initialization : Complete -etcd Install : Complete -NFS Install : Not Started -Load balancer Install : Not Started -Master Install : Complete -Master Additional Install : Complete -Node Install : Complete -GlusterFS Install : Not Started -Hosted Install : Complete -Metrics Install : Not Started -Logging Install : Not Started -Prometheus Install : Not Started -Service Catalog Install : Not Started - ------------------------------------------------------ -Example display if a failure occurs during execution: - -INSTALLER STATUS *************************************************************** -Initialization : Complete -etcd Install : Complete -NFS Install : Not Started -Load balancer Install : Not Started -Master Install : In Progress - This phase can be restarted by running: playbooks/byo/openshift-master/config.yml -Master Additional Install : Not Started -Node Install : Not Started -GlusterFS Install : Not Started -Hosted Install : Not Started -Metrics Install : Not Started -Logging Install : Not Started -Prometheus Install : Not Started -Service Catalog Install : Not Started - -''' - class CallbackModule(CallbackBase): """This callback summarizes installation phase status.""" @@ -163,9 +113,10 @@ class CallbackModule(CallbackBase): phase_title = phase_attributes[phase]['title'] padding = max_column - len(phase_title) + 2 if phase in stats.custom['_run']: - phase_status = stats.custom['_run'][phase] + phase_status = stats.custom['_run'][phase]['status'] + phase_time = phase_time_delta(stats.custom['_run'][phase]) self._display.display( - '{}{}: {}'.format(phase_title, ' ' * padding, phase_status), + '{}{}: {} ({})'.format(phase_title, ' ' * padding, phase_status, phase_time), color=self.phase_color(phase_status)) if phase_status == 'In Progress' and phase != 'installer_phase_initialize': self._display.display( @@ -192,3 +143,17 @@ class CallbackModule(CallbackBase): phase_color = C.COLOR_WARN return phase_color + + +def phase_time_delta(phase): + """ Calculate the difference between phase start and end times """ + time_format = '%Y%m%d%H%M%SZ' + phase_start = datetime.strptime(phase['start'], time_format) + if 'end' not in phase: + # The phase failed so set the end time to now + phase_end = datetime.now() + else: + phase_end = datetime.strptime(phase['end'], time_format) + delta = str(phase_end - phase_start).split(".")[0] # Trim microseconds + + return delta |