Merge pull request #1155 from mwoodson/host_monitoring

merging master into prod

10 files changed, 88 insertions, 91 deletions

diff --git a/playbooks/aws/openshift-cluster/cluster_hosts.yml b/playbooks/aws/openshift-cluster/cluster_hosts.yml
new file mode 100644
index 000000000..d6b413c6f
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/cluster_hosts.yml
@@ -0,0 +1,22 @@
+---
+g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))
+                     | intersect((groups['tag_clusterid_' ~ cluster_id]|default([])))
+                     | intersect((groups['tag_environment_' ~ cluster_env]|default([]))) }}"
+
+g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))
+                     | intersect((groups['tag_clusterid_' ~ cluster_id]|default([])))
+                     | intersect((groups['tag_environment_' ~ cluster_env]|default([]))) }}"
+
+g_master_hosts: "{{ (groups['tag_host-type_master']|default([]))
+                     | intersect((groups['tag_clusterid_' ~ cluster_id]|default([])))
+                     | intersect((groups['tag_environment_' ~ cluster_env]|default([]))) }}"
+
+g_node_hosts:   "{{ (groups['tag_host-type_node']|default([]))
+                     | intersect((groups['tag_clusterid_' ~ cluster_id]|default([])))
+                     | intersect((groups['tag_environment_' ~ cluster_env]|default([]))) }}"
+
+g_nfs_hosts:    "{{ (groups['tag_host-type_nfs']|default([]))
+                   | intersect((groups['tag_environment_' ~ cluster_id]|default([]))) }}"
+
+g_all_hosts:    "{{ g_master_hosts | union(g_node_hosts) | union(g_etcd_hosts)
+                    | union(g_lb_hosts) | default([]) }}"
diff --git a/playbooks/aws/openshift-cluster/config.yml b/playbooks/aws/openshift-cluster/config.yml
index 50fe42d6c..abdb23d78 100644
--- a/playbooks/aws/openshift-cluster/config.yml
+++ b/playbooks/aws/openshift-cluster/config.yml
@@ -1,26 +1,14 @@
 ---
-- hosts: localhost
-  gather_facts: no
-  connection: local
-  become: no
-  vars_files:
-  - vars.yml
-  tasks:
-  - set_fact:
-      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
-      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
-
 - include: ../../common/openshift-cluster/config.yml
+  vars_files:
+  - ../../aws/openshift-cluster/vars.yml
+  - ../../aws/openshift-cluster/cluster_hosts.yml
   vars:
-    g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))     | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_node_hosts:   "{{ (groups['tag_host-type_node']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
-    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
+    g_ssh_user:     "{{ deployment_vars[deployment_type].ssh_user }}"
+    g_sudo:         "{{ deployment_vars[deployment_type].sudo }}"
     g_nodeonmaster: true
     openshift_cluster_id: "{{ cluster_id }}"
-    openshift_debug_level: 2
+    openshift_debug_level: "{{ debug_level }}"
     openshift_deployment_type: "{{ deployment_type }}"
     openshift_hostname: "{{ ec2_private_ip_address }}"
     openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/list.yml b/playbooks/aws/openshift-cluster/list.yml
index 8341ba9c1..8b41a355e 100644
--- a/playbooks/aws/openshift-cluster/list.yml
+++ b/playbooks/aws/openshift-cluster/list.yml
@@ -7,7 +7,7 @@
   vars_files:
   - vars.yml
   tasks:
-  - set_fact: scratch_group=tag_env_{{ cluster_id }}
+  - set_fact: scratch_group=tag_clusterid_{{ cluster_id }}
     when: cluster_id != ''
   - set_fact: scratch_group=all
     when: cluster_id == ''
diff --git a/playbooks/aws/openshift-cluster/scaleup.yml b/playbooks/aws/openshift-cluster/scaleup.yml
index 9c9118286..c2135cd03 100644
--- a/playbooks/aws/openshift-cluster/scaleup.yml
+++ b/playbooks/aws/openshift-cluster/scaleup.yml
@@ -7,9 +7,6 @@
   vars_files:
   - vars.yml
   tasks:
-  - set_fact:
-      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
-      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
   - name: Evaluate oo_hosts_to_update
     add_host:
       name: "{{ item }}"
@@ -21,16 +18,16 @@
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 
 - include: ../../common/openshift-cluster/scaleup.yml
+  vars_files:
+  - ../../aws/openshift-cluster/vars.yml
+  - ../../aws/openshift-cluster/cluster_hosts.yml
   vars:
-    g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))     | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
     g_new_node_hosts: "{{ groups.nodes_to_add }}"
-    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
-    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
+    g_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+    g_sudo: "{{ deployment_vars[deployment_type].sudo }}"
     g_nodeonmaster: true
     openshift_cluster_id: "{{ cluster_id }}"
-    openshift_debug_level: 2
+    openshift_debug_level: "{{ debug_level }}"
     openshift_deployment_type: "{{ deployment_type }}"
     openshift_hostname: "{{ ec2_private_ip_address }}"
     openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/service.yml b/playbooks/aws/openshift-cluster/service.yml
index ce0992a45..d5f7d6b19 100644
--- a/playbooks/aws/openshift-cluster/service.yml
+++ b/playbooks/aws/openshift-cluster/service.yml
@@ -6,6 +6,7 @@
   gather_facts: no
   vars_files:
   - vars.yml
+  - cluster_hosts.yml
   tasks:
   - fail: msg="cluster_id is required to be injected in this playbook"
     when: cluster_id is not defined
@@ -16,7 +17,7 @@
       groups: g_service_masters
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: "{{ g_master_hosts | default([]) }}"
+    with_items: "{{ master_hosts | default([]) }}"
 
   - name: Evaluate g_service_nodes
     add_host:
@@ -24,7 +25,7 @@
       groups: g_service_nodes
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: "{{ g_node_hosts | default([]) }}"
+    with_items: "{{ node_hosts | default([]) }}"
 
 - include: ../../common/openshift-node/service.yml
 - include: ../../common/openshift-master/service.yml
diff --git a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
index 1fbd71a75..6090ed6fe 100644
--- a/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
+++ b/playbooks/aws/openshift-cluster/tasks/launch_instances.yml
@@ -2,7 +2,8 @@
 - set_fact:
     created_by: "{{ lookup('env', 'LOGNAME')|default(cluster, true) }}"
     docker_vol_ephemeral: "{{ lookup('env', 'os_docker_vol_ephemeral') | default(false, true) }}"
-    env: "{{ cluster }}"
+    cluster: "{{ cluster_id }}"
+    env: "{{ cluster_env }}"
     host_type: "{{ type }}"
     sub_host_type: "{{ g_sub_host_type }}"
 
@@ -123,7 +124,8 @@
     wait: yes
     instance_tags:
       created-by: "{{ created_by }}"
-      env: "{{ env }}"
+      clusterid: "{{ cluster }}"
+      environment: "{{ cluster_env }}"
       host-type: "{{ host_type }}"
       sub-host-type: "{{ sub_host_type }}"
     volumes: "{{ volumes }}"
@@ -139,7 +141,8 @@
       Name: "{{ item.0 }}"
 
 - set_fact:
-    instance_groups: "tag_created-by_{{ created_by }}, tag_env_{{ env }}, tag_host-type_{{ host_type }}, tag_sub-host-type_{{ sub_host_type }}"
+    instance_groups: "tag_created-by_{{ created_by }}, tag_clusterid_{{ cluster }}, tag_environment_{{ cluster_env }},
+                    tag_host-type_{{ host_type }}, tag_sub-host-type_{{ sub_host_type }}"
 
 - set_fact:
     node_label:
diff --git a/playbooks/aws/openshift-cluster/terminate.yml b/playbooks/aws/openshift-cluster/terminate.yml
index aafd40c43..4b9c80b14 100644
--- a/playbooks/aws/openshift-cluster/terminate.yml
+++ b/playbooks/aws/openshift-cluster/terminate.yml
@@ -7,13 +7,12 @@
   vars_files:
   - vars.yml
   tasks:
-  - set_fact: scratch_group=tag_env_{{ cluster_id }}
   - add_host:
       name: "{{ item }}"
       groups: oo_hosts_to_terminate
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: groups[scratch_group] | default([]) | difference(['localhost'])
+    with_items: (groups['tag_clusterid_' ~ cluster_id] | default([])) | difference(['localhost'])
 
 - name: Unsubscribe VMs
   hosts: oo_hosts_to_terminate
@@ -29,34 +28,35 @@
   connection: local
   become: no
   gather_facts: no
-  vars:
-    host_vars: "{{ hostvars
-                   | oo_select_keys(groups['oo_hosts_to_terminate']) }}"
   tasks:
     - name: Remove tags from instances
-      ec2_tag: resource={{ item.ec2_id }} region={{ item.ec2_region }} state=absent
-      args:
+      ec2_tag:
+        resource: "{{ hostvars[item]['ec2_id'] }}"
+        region: "{{ hostvars[item]['ec2_region'] }}"
+        state: absent
         tags:
-          env: "{{ item['ec2_tag_env'] }}"
-          host-type: "{{ item['ec2_tag_host-type'] }}"
-          sub_host_type: "{{ item['ec2_tag_sub-host-type'] }}"
-      with_items: host_vars
+          environment:   "{{ hostvars[item]['ec2_tag_environment'] }}"
+          clusterid:     "{{ hostvars[item]['ec2_tag_clusterid'] }}"
+          host-type:     "{{ hostvars[item]['ec2_tag_host-type'] }}"
+          sub_host_type: "{{ hostvars[item]['ec2_tag_sub-host-type'] }}"
+      with_items: groups.oo_hosts_to_terminate
       when: "'oo_hosts_to_terminate' in groups"
 
     - name: Terminate instances
       ec2:
         state: absent
-        instance_ids: ["{{ item.ec2_id }}"]
-        region: "{{ item.ec2_region }}"
+        instance_ids: ["{{ hostvars[item].ec2_id }}"]
+        region: "{{ hostvars[item].ec2_region }}"
       ignore_errors: yes
       register: ec2_term
-      with_items: host_vars
+      with_items: groups.oo_hosts_to_terminate
       when: "'oo_hosts_to_terminate' in groups"
 
     # Fail if any of the instances failed to terminate with an error other
     # than 403 Forbidden
-    - fail: msg=Terminating instance {{ item.ec2_id }} failed with message {{ item.msg }}
-      when: "'oo_hosts_to_terminate' in groups and item.failed and not item.msg | search(\"error: EC2ResponseError: 403 Forbidden\")"
+    - fail:
+        msg: "Terminating instance {{ item.ec2_id }} failed with message {{ item.msg }}"
+      when: "'oo_hosts_to_terminate' in groups and item.has_key('failed') and item.failed"
       with_items: ec2_term.results
 
     - name: Stop instance if termination failed
@@ -65,7 +65,7 @@
         instance_ids: ["{{ item.item.ec2_id }}"]
         region: "{{ item.item.ec2_region }}"
       register: ec2_stop
-      when: "'oo_hosts_to_terminate' in groups and item.failed"
+      when: "'oo_hosts_to_terminate' in groups and item.has_key('failed') and item.failed"
       with_items: ec2_term.results
 
     - name: Rename stopped instances
diff --git a/playbooks/aws/openshift-cluster/update.yml b/playbooks/aws/openshift-cluster/update.yml
index 3df0c3f3a..32bab76b5 100644
--- a/playbooks/aws/openshift-cluster/update.yml
+++ b/playbooks/aws/openshift-cluster/update.yml
@@ -4,13 +4,9 @@
   connection: local
   become: no
   gather_facts: no
-  vars:
-    g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([])) | intersect(groups['tag_env_' ~ cluster_id]) }}"
-    g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([])) | intersect(groups['tag_env_' ~ cluster_id]) }}"
-    g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect(groups['tag_env_' ~ cluster_id]) }}"
-    g_node_hosts:   "{{ (groups['tag_host-type_node']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
   vars_files:
   - vars.yml
+  - cluster_hosts.yml
   tasks:
   - name: Update - Evaluate oo_hosts_to_update
     add_host:
@@ -18,7 +14,7 @@
       groups: oo_hosts_to_update
       ansible_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
       ansible_sudo: "{{ deployment_vars[deployment_type].sudo }}"
-    with_items: "{{ g_master_hosts | union(g_node_hosts) | union(g_etcd_hosts) | default([]) }}"
+    with_items: "{{ g_all_hosts | default([]) }}"
 
 - include: ../../common/openshift-cluster/update_repos_and_packages.yml
 
diff --git a/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml b/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
index 20cc97c8a..11026e38d 100644
--- a/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+++ b/playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
@@ -2,32 +2,16 @@
 # This playbook upgrades an existing AWS cluster, leaving nodes untouched if used with an 'online' deployment type.
 # Usage:
 #  ansible-playbook playbooks/aws/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml -e deployment_type=online -e cluster_id=<cluster_id>
-- hosts: localhost
-  gather_facts: no
-  vars_files:
-  - ../../vars.yml
-  - "../../vars.{{ deployment_type }}.{{ cluster_id }}.yml"
-
-  tasks:
-  - set_fact:
-      g_ssh_user_tmp: "{{ deployment_vars[deployment_type].ssh_user }}"
-      g_sudo_tmp: "{{ deployment_vars[deployment_type].sudo }}"
-
-  - set_fact:
-      tmp_nodes_group: "{{ 'tag_env-host-type_' ~ cluster_id ~ '-openshift-node' }}"
-    when: deployment_type != 'online'
-
 - include: ../../../../common/openshift-cluster/upgrades/v3_0_to_v3_1/upgrade.yml
+  vars_files:
+  - "{{lookup('file', '../../../../aws/openshift-cluster/vars.yml')}}"
+  - "{{lookup('file', '../../../../aws/openshift-cluster/cluster_hosts.yml')}}"
   vars:
-    g_etcd_hosts:   "{{ (groups['tag_host-type_etcd']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_lb_hosts:     "{{ (groups['tag_host-type_lb']|default([]))     | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_master_hosts: "{{ (groups['tag_host-type_master']|default([])) | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_node_hosts:   "{{ (groups['tag_host-type_node']|default([]))   | intersect((groups['tag_env_' ~ cluster_id]|default([]))) }}"
-    g_ssh_user: "{{ hostvars.localhost.g_ssh_user_tmp }}"
-    g_sudo: "{{ hostvars.localhost.g_sudo_tmp }}"
+    g_ssh_user: "{{ deployment_vars[deployment_type].ssh_user }}"
+    g_sudo: "{{ deployment_vars[deployment_type].sudo }}"
     g_nodeonmaster: true
     openshift_cluster_id: "{{ cluster_id }}"
-    openshift_debug_level: 2
+    openshift_debug_level: "{{ debug_level }}"
     openshift_deployment_type: "{{ deployment_type }}"
     openshift_hostname: "{{ ec2_private_ip_address }}"
     openshift_public_hostname: "{{ ec2_ip_address }}"
diff --git a/playbooks/aws/openshift-cluster/vars.yml b/playbooks/aws/openshift-cluster/vars.yml
index 95bc4b3e2..ae12286bd 100644
--- a/playbooks/aws/openshift-cluster/vars.yml
+++ b/playbooks/aws/openshift-cluster/vars.yml
@@ -1,8 +1,23 @@
 ---
+debug_level: 2
+
+deployment_rhel7_ent_base:
+  # rhel-7.1, requires cloud access subscription
+  image: ami-10663b78
+  image_name:
+  region: us-east-1
+  ssh_user: ec2-user
+  sudo: yes
+  keypair: libra
+  type: m4.large
+  security_groups: [ 'public' ]
+  vpc_subnet:
+  assign_public_ip:
+
 deployment_vars:
   origin:
     # centos-7, requires marketplace
-    image: ami-96a818fe
+    image: ami-61bbf104
     image_name:
     region: us-east-1
     ssh_user: centos
@@ -24,15 +39,6 @@ deployment_vars:
     security_groups: [ 'public' ]
     vpc_subnet:
     assign_public_ip:
-  enterprise:
-    # rhel-7.1, requires cloud access subscription
-    image: ami-10663b78
-    image_name:
-    region: us-east-1
-    ssh_user: ec2-user
-    sudo: yes
-    keypair: libra
-    type: m4.large
-    security_groups: [ 'public' ]
-    vpc_subnet:
-    assign_public_ip:
+  enterprise: "{{ deployment_rhel7_ent_base }}"
+  openshift-enterprise: "{{ deployment_rhel7_ent_base }}"
+  atomic-enterprise: "{{ deployment_rhel7_ent_base }}"