Merge pull request #5948 from mtnbikenc/checkpoint-times

Automatic merge from submit-queue.

Add execution times to checkpoint status

This PR adds execution times to the checkpoint status report at the end of an installer playbook run.

- Checkpoint plays now use `all` host group to avoid using an undefined host group during the Initialization phase as well as maintaining the need to collect inventory vars for host group conditionals
- `run_once: true` is used to limit the execution to only one host
- `installer_phase_*` data is now stored as a dict so multiple data points can be stored for that phase
- Documentation updated


```
INSTALLER STATUS ***************************************************************
Initialization             : Complete (0:02:14)
Health Check               : Complete (0:01:10)
etcd Install               : Complete (0:02:01)
Master Install             : Complete (0:11:43)
Master Additional Install  : Complete (0:00:54)
Node Install               : Complete (0:14:11)
Hosted Install             : Complete (0:03:28)
```

15 files changed, 150 insertions, 90 deletions

diff --git a/playbooks/common/openshift-checks/install.yml b/playbooks/common/openshift-checks/install.yml
index 6701a2e15..93cf6c359 100644
--- a/playbooks/common/openshift-checks/install.yml
+++ b/playbooks/common/openshift-checks/install.yml
@@ -1,13 +1,15 @@
 ---
 - name: Health Check Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Health Check 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_health: "In Progress"
-      aggregate: false
+        installer_phase_health:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Health Checks
   hosts: oo_all_hosts
@@ -37,11 +39,13 @@
       - docker_image_availability
 
 - name: Health Check Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Health Check 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_health: "Complete"
-      aggregate: false
+        installer_phase_health:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/openshift_hosted.yml b/playbooks/common/openshift-cluster/openshift_hosted.yml
index 281ccce2e..15ee60dc0 100644
--- a/playbooks/common/openshift-cluster/openshift_hosted.yml
+++ b/playbooks/common/openshift-cluster/openshift_hosted.yml
@@ -1,13 +1,15 @@
 ---
 - name: Hosted Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Hosted install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_hosted: "In Progress"
-      aggregate: false
+        installer_phase_hosted:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: create_persistent_volumes.yml
 
@@ -30,11 +32,13 @@
   - openshift_crio_enable_docker_gc | default(False) | bool
 
 - name: Hosted Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Hosted install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_hosted: "Complete"
-      aggregate: false
+        installer_phase_hosted:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/openshift_logging.yml b/playbooks/common/openshift-cluster/openshift_logging.yml
index 529a4c939..bc59bd95a 100644
--- a/playbooks/common/openshift-cluster/openshift_logging.yml
+++ b/playbooks/common/openshift-cluster/openshift_logging.yml
@@ -1,13 +1,15 @@
 ---
 - name: Logging Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Logging install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_logging: "In Progress"
-      aggregate: false
+        installer_phase_logging:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Aggregated Logging
   hosts: oo_first_master
@@ -23,11 +25,13 @@
         tasks_from: update_master_config
 
 - name: Logging Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Logging install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_logging: "Complete"
-      aggregate: false
+        installer_phase_logging:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/openshift_metrics.yml b/playbooks/common/openshift-cluster/openshift_metrics.yml
index 9c0bd489b..80cd93e5f 100644
--- a/playbooks/common/openshift-cluster/openshift_metrics.yml
+++ b/playbooks/common/openshift-cluster/openshift_metrics.yml
@@ -1,13 +1,15 @@
 ---
 - name: Metrics Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Metrics install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_metrics: "In Progress"
-      aggregate: false
+        installer_phase_metrics:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: OpenShift Metrics
   hosts: oo_first_master
@@ -24,11 +26,13 @@
       tasks_from: update_master_config.yaml
 
 - name: Metrics Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Metrics install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_metrics: "Complete"
-      aggregate: false
+        installer_phase_metrics:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/openshift_prometheus.yml b/playbooks/common/openshift-cluster/openshift_prometheus.yml
index a73b294a5..7aa9a16e6 100644
--- a/playbooks/common/openshift-cluster/openshift_prometheus.yml
+++ b/playbooks/common/openshift-cluster/openshift_prometheus.yml
@@ -1,13 +1,15 @@
 ---
 - name: Prometheus Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Prometheus install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_prometheus: "In Progress"
-      aggregate: false
+        installer_phase_prometheus:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Create Hosted Resources - openshift_prometheus
   hosts: oo_first_master
@@ -15,11 +17,13 @@
   - role: openshift_prometheus
 
 - name: Prometheus Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Prometheus install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_prometheus: "Complete"
-      aggregate: false
+        installer_phase_prometheus:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/service_catalog.yml b/playbooks/common/openshift-cluster/service_catalog.yml
index bd964b2ce..7bb8511f6 100644
--- a/playbooks/common/openshift-cluster/service_catalog.yml
+++ b/playbooks/common/openshift-cluster/service_catalog.yml
@@ -1,13 +1,15 @@
 ---
 - name: Service Catalog Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Service Catalog install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_servicecatalog: "In Progress"
-      aggregate: false
+        installer_phase_servicecatalog:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Service Catalog
   hosts: oo_first_master
@@ -19,11 +21,13 @@
     first_master: "{{ groups.oo_first_master[0] }}"
 
 - name: Service Catalog Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Service Catalog install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_servicecatalog: "Complete"
-      aggregate: false
+        installer_phase_servicecatalog:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-cluster/std_include.yml b/playbooks/common/openshift-cluster/std_include.yml
index 45b34c8bd..fe376fe31 100644
--- a/playbooks/common/openshift-cluster/std_include.yml
+++ b/playbooks/common/openshift-cluster/std_include.yml
@@ -1,15 +1,17 @@
 ---
 - name: Initialization Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   roles:
   - installer_checkpoint
   tasks:
   - name: Set install initialization 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "In Progress"
-      aggregate: false
+        installer_phase_initialize:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: evaluate_groups.yml
   tags:
@@ -36,11 +38,13 @@
   - always
 
 - name: Initialization Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set install initialization 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_initialize: "Complete"
-      aggregate: false
+        installer_phase_initialize:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-etcd/config.yml b/playbooks/common/openshift-etcd/config.yml
index 48d46bbb0..3fe483785 100644
--- a/playbooks/common/openshift-etcd/config.yml
+++ b/playbooks/common/openshift-etcd/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: etcd Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set etcd install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_etcd: "In Progress"
-      aggregate: false
+        installer_phase_etcd:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: ca.yml
 
@@ -26,11 +28,13 @@
   - role: nickhammond.logrotate
 
 - name: etcd Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set etcd install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_etcd: "Complete"
-      aggregate: false
+        installer_phase_etcd:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-glusterfs/config.yml b/playbooks/common/openshift-glusterfs/config.yml
index c2ae5f313..19e14ab3e 100644
--- a/playbooks/common/openshift-glusterfs/config.yml
+++ b/playbooks/common/openshift-glusterfs/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: GlusterFS Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set GlusterFS install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_glusterfs: "In Progress"
-      aggregate: false
+        installer_phase_glusterfs:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Open firewall ports for GlusterFS nodes
   hosts: glusterfs
@@ -46,11 +48,13 @@
     when: groups.oo_glusterfs_to_config | default([]) | count > 0
 
 - name: GlusterFS Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set GlusterFS install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_glusterfs: "Complete"
-      aggregate: false
+        installer_phase_glusterfs:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml
index 2a703cb61..d737b836b 100644
--- a/playbooks/common/openshift-loadbalancer/config.yml
+++ b/playbooks/common/openshift-loadbalancer/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: Load Balancer Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set load balancer install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_loadbalancer: "In Progress"
-      aggregate: false
+        installer_phase_loadbalancer:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Configure firewall and docker for load balancers
   hosts: oo_lb_to_config:!oo_masters_to_config:!oo_nodes_to_config
@@ -37,11 +39,13 @@
   - role: tuned
 
 - name: Load Balancer Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set load balancer install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_loadbalancer: "Complete"
-      aggregate: false
+        installer_phase_loadbalancer:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-management/config.yml b/playbooks/common/openshift-management/config.yml
index 908679e81..3f1cdf713 100644
--- a/playbooks/common/openshift-management/config.yml
+++ b/playbooks/common/openshift-management/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: Management Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Management install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_management: "In Progress"
-      aggregate: false
+        installer_phase_management:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Setup CFME
   hosts: oo_first_master
@@ -25,11 +27,13 @@
       template_dir: "{{ hostvars[groups.masters.0].r_openshift_management_mktemp.stdout }}"
 
 - name: Management Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Management install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_management: "Complete"
-      aggregate: false
+        installer_phase_management:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-master/additional_config.yml b/playbooks/common/openshift-master/additional_config.yml
index 350557f19..4fef5b923 100644
--- a/playbooks/common/openshift-master/additional_config.yml
+++ b/playbooks/common/openshift-master/additional_config.yml
@@ -1,13 +1,15 @@
 ---
 - name: Master Additional Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master Additional install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master_additional: "In Progress"
-      aggregate: false
+        installer_phase_master_additional:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Additional master configuration
   hosts: oo_first_master
@@ -36,11 +38,13 @@
     when: openshift_use_flannel | default(false) | bool
 
 - name: Master Additional Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master Additional install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master_additional: "Complete"
-      aggregate: false
+        installer_phase_master_additional:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 7ce0362ef..6b0fd6b7c 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: Master Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master: "In Progress"
-      aggregate: false
+        installer_phase_master:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: certificates.yml
 
@@ -238,11 +240,13 @@
     r_openshift_excluder_service_type: "{{ openshift.common.service_type }}"
 
 - name: Master Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Master install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_master: "Complete"
-      aggregate: false
+        installer_phase_master:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-nfs/config.yml b/playbooks/common/openshift-nfs/config.yml
index ce672daf5..6ea77e00b 100644
--- a/playbooks/common/openshift-nfs/config.yml
+++ b/playbooks/common/openshift-nfs/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: NFS Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set NFS install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_nfs: "In Progress"
-      aggregate: false
+        installer_phase_nfs:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - name: Configure nfs
   hosts: oo_nfs_to_config
@@ -16,11 +18,13 @@
   - role: openshift_storage_nfs
 
 - name: NFS Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set NFS install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_nfs: "Complete"
-      aggregate: false
+        installer_phase_nfs:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 4f8f98aef..28e3c1b1b 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -1,13 +1,15 @@
 ---
 - name: Node Install Checkpoint Start
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Node install 'In Progress'
+    run_once: true
     set_stats:
       data:
-        installer_phase_node: "In Progress"
-      aggregate: false
+        installer_phase_node:
+          status: "In Progress"
+          start: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"
 
 - include: certificates.yml
 
@@ -24,11 +26,13 @@
 - include: enable_excluders.yml
 
 - name: Node Install Checkpoint End
-  hosts: oo_all_hosts
+  hosts: all
   gather_facts: false
   tasks:
   - name: Set Node install 'Complete'
+    run_once: true
     set_stats:
       data:
-        installer_phase_node: "Complete"
-      aggregate: false
+        installer_phase_node:
+          status: "Complete"
+          end: "{{ lookup('pipe', 'date +%Y%m%d%H%M%SZ') }}"