openshift_facts role/module refactor default settings

- Add openshift_facts role and module
  - Created new role openshift_facts that contains an openshift_facts module
  - Refactor openshift_* roles to use openshift_facts instead of relying on
    defaults
  - Refactor playbooks to use openshift_facts
  - Cleanup inventory group_vars

- Update defaults
  - update openshift_master role firewall defaults
    - remove etcd peer port, since we will not be supporting clustered embedded
      etcd
    - remove 8444 since console now runs on the api port by default
    - add 8444 and 7001 to disabled services to ensure removal if updating

- Add new role os_env_extras_node that is a subset of the docker role
  - previously, we were starting/enabling docker which was causing issues with some
  installations
  - Does not install or start docker, since the openshift-node role will
    handle that for us
  - Only adds root to the dockerroot group
  - Update playbooks to use ops_env_extras_node role instead of docker role

- os_firewall bug fixes
  - ignore ip6tables for now, since we are not configuring any ipv6 rules
  - if installing package do a daemon-reload before starting/enabling service

- Add aws support to bin/cluster

- Add list action to bin/cluster

- Add update action to bin/cluster

- cleanup some stray debug statements

- some variable renaming for clarity

16 files changed, 393 insertions, 66 deletions

diff --git a/playbooks/aws/openshift-cluster/filter_plugins b/playbooks/aws/openshift-cluster/filter_plugins
new file mode 120000
index 000000000..99a95e4ca
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/filter_plugins
@@ -0,0 +1 @@
+../../../filter_plugins
\ No newline at end of file
diff --git a/playbooks/aws/openshift-cluster/launch.yml b/playbooks/aws/openshift-cluster/launch.yml
new file mode 100644
index 000000000..3561c1803
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/launch.yml
@@ -0,0 +1,62 @@
+---
+- name: Launch instance(s)
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars_files:
+      - vars.yml
+  tasks:
+    - set_fact: k8s_type="master"
+
+    - name: Generate master instance names(s)
+      set_fact: scratch={{ cluster_id }}-{{ k8s_type }}-{{ '%05x' |format( 1048576 |random) }}
+      register: master_names_output
+      with_sequence: start=1 end={{ num_masters }}
+
+    # These set_fact's cannot be combined
+    - set_fact:
+        master_names_string: "{% for item in master_names_output.results %}{{ item.ansible_facts.scratch }} {% endfor %}"
+
+    - set_fact:
+        master_names: "{{ master_names_string.strip().split(' ') }}"
+
+    - include: launch_instances.yml
+      vars:
+        instances: "{{ master_names }}"
+        cluster: "{{ cluster_id }}"
+        type: "{{ k8s_type }}"
+
+    - set_fact: k8s_type="node"
+
+    - name: Generate node instance names(s)
+      set_fact: scratch={{ cluster_id }}-{{ k8s_type }}-{{ '%05x' |format( 1048576 |random) }}
+      register: node_names_output
+      with_sequence: start=1 end={{ num_nodes }}
+
+    # These set_fact's cannot be combined
+    - set_fact:
+        node_names_string: "{% for item in node_names_output.results %}{{ item.ansible_facts.scratch }} {% endfor %}"
+
+    - set_fact:
+        node_names: "{{ node_names_string.strip().split(' ') }}"
+
+    - include: launch_instances.yml
+      vars:
+        instances: "{{ node_names }}"
+        cluster: "{{ cluster_id }}"
+        type: "{{ k8s_type }}"
+
+- hosts: "tag_env_{{ cluster_id }}"
+  roles:
+  - openshift_repos
+  - os_update_latest
+
+- include: ../openshift-master/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type_{{ cluster_id }}-openshift-master\"]"
+
+- include: ../openshift-node/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type_{{ cluster_id }}-openshift-node\"]"
+
+- include: list.yml
diff --git a/playbooks/aws/openshift-cluster/launch_instances.yml b/playbooks/aws/openshift-cluster/launch_instances.yml
new file mode 100644
index 000000000..e4d5952fd
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/launch_instances.yml
@@ -0,0 +1,62 @@
+---
+- set_fact:
+    machine_type: "{{ lookup('env', 'ec2_instance_type')|default('m3.large', true) }}"
+    machine_image: "{{ lookup('env', 'ec2_ami')|default('ami-307b3658', true) }}"
+    machine_region: "{{ lookup('env', 'ec2_region')|default('us-east-1', true) }}"
+    machine_keypair: "{{ lookup('env', 'ec2_keypair')|default('libra', true) }}"
+    created_by: "{{ lookup('env', 'LOGNAME')|default(cluster, true) }}"
+    env: "{{ cluster }}"
+    host_type: "{{ type }}"
+    env_host_type: "{{ cluster }}-openshift-{{ type }}"
+
+- name: Launch instance(s)
+  ec2:
+    state: present
+    region: "{{ machine_region }}"
+    keypair: "{{ machine_keypair }}"
+    group: ['public']
+    instance_type: "{{ machine_type }}"
+    image: "{{ machine_image }}"
+    count: "{{ instances | oo_len }}"
+    wait: yes
+    instance_tags:
+      created-by: "{{ created_by }}"
+      env: "{{ env }}"
+      host-type: "{{ host_type }}"
+      env-host-type: "{{ env_host_type }}"
+  register: ec2
+
+- name: Add Name tag to instances
+  ec2_tag: resource={{ item.1.id }} region={{ machine_region }} state=present
+  with_together:
+  - instances
+  - ec2.instances
+  args:
+    tags:
+      Name: "{{ item.0 }}"
+
+- set_fact:
+    instance_groups: tag_created-by_{{ created_by }}, tag_env_{{ env }}, tag_host-type_{{ host_type }}, tag_env-host-type_{{ env_host_type }}
+
+- name: Add new instances groups and variables
+  add_host:
+    hostname: "{{ item.0 }}"
+    ansible_ssh_host: "{{ item.1.dns_name }}"
+    groups: "{{ instance_groups }}"
+    ec2_private_ip_address: "{{ item.1.private_ip }}"
+    ec2_ip_address: "{{ item.1.public_ip }}"
+  with_together:
+  - instances
+  - ec2.instances
+
+- name: Wait for ssh
+  wait_for: "port=22 host={{ item.dns_name }}"
+  with_items: ec2.instances
+
+- name: Wait for root user setup
+  command: "ssh -o StrictHostKeyChecking=no -o PasswordAuthentication=no -o ConnectTimeout=10 -o UserKnownHostsFile=/dev/null root@{{ item.dns_name }} echo root user is setup"
+  register: result
+  until: result.rc == 0
+  retries: 20
+  delay: 10
+  with_items: ec2.instances
diff --git a/playbooks/aws/openshift-cluster/list.yml b/playbooks/aws/openshift-cluster/list.yml
new file mode 100644
index 000000000..08e9e2df4
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/list.yml
@@ -0,0 +1,17 @@
+---
+- name: Generate oo_list_hosts group
+  hosts: localhost
+  gather_facts: no
+  tasks:
+  - set_fact: scratch_group=tag_env_{{ cluster_id }}
+    when: cluster_id != ''
+  - set_fact: scratch_group=all
+    when: scratch_group is not defined
+  - add_host: name={{ item }} groups=oo_list_hosts
+    with_items: groups[scratch_group] | difference(['localhost'])
+
+- name: List Hosts
+  hosts: oo_list_hosts
+  gather_facts: no
+  tasks:
+  - debug: msg="public:{{hostvars[inventory_hostname].ec2_ip_address}} private:{{hostvars[inventory_hostname].ec2_private_ip_address}}"
diff --git a/playbooks/aws/openshift-cluster/roles b/playbooks/aws/openshift-cluster/roles
new file mode 120000
index 000000000..20c4c58cf
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/roles
@@ -0,0 +1 @@
+../../../roles
\ No newline at end of file
diff --git a/playbooks/aws/openshift-cluster/terminate.yml b/playbooks/aws/openshift-cluster/terminate.yml
new file mode 100644
index 000000000..39607633a
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/terminate.yml
@@ -0,0 +1,14 @@
+---
+- name: Terminate instance(s)
+  hosts: localhost
+
+  vars_files:
+    - vars.yml
+
+- include: ../openshift-node/terminate.yml
+  vars:
+    oo_host_group_exp: 'groups["tag_env-host-type_{{ cluster_id }}-openshift-node"]'
+
+- include: ../openshift-master/terminate.yml
+  vars:
+    oo_host_group_exp: 'groups["tag_env-host-type_{{ cluster_id }}-openshift-master"]'
diff --git a/playbooks/aws/openshift-cluster/update.yml b/playbooks/aws/openshift-cluster/update.yml
new file mode 100644
index 000000000..90ecdc6ab
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/update.yml
@@ -0,0 +1,13 @@
+---
+- hosts: "tag_env_{{ cluster_id }}"
+  roles:
+  - openshift_repos
+  - os_update_latest
+
+- include: ../openshift-master/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type_{{ cluster_id }}-openshift-master\"]"
+
+- include: ../openshift-node/config.yml
+  vars:
+    oo_host_group_exp: "groups[\"tag_env-host-type_{{ cluster_id }}-openshift-node\"]"
diff --git a/playbooks/aws/openshift-cluster/vars.yml b/playbooks/aws/openshift-cluster/vars.yml
new file mode 100644
index 000000000..ed97d539c
--- /dev/null
+++ b/playbooks/aws/openshift-cluster/vars.yml
@@ -0,0 +1 @@
+---
diff --git a/playbooks/aws/openshift-master/config.yml b/playbooks/aws/openshift-master/config.yml
index bbf1f654a..1c4060eee 100644
--- a/playbooks/aws/openshift-master/config.yml
+++ b/playbooks/aws/openshift-master/config.yml
@@ -1,5 +1,5 @@
 ---
-- name: "populate oo_masters_to_config host group if needed"
+- name: Populate oo_masters_to_config host group if needed
   hosts: localhost
   gather_facts: no
   tasks:
@@ -8,34 +8,17 @@
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
 
-- name: "Gather facts for nodes in {{ oo_env }}"
-  hosts: "tag_env-host-type_{{ oo_env }}-openshift-node"
-  connection: ssh
-  user: root
-
-- name: "Set Origin specific facts on localhost (for later use)"
-  hosts: localhost
-  gather_facts: no
-  tasks:
-    - name: Setting openshift_node_ips fact on localhost
-      set_fact:
-        openshift_node_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type_' + oo_env + '-openshift-node'])
-            | oo_collect(attribute='ansible_default_ipv4.address') }}"
-      when: groups['tag_env-host-type_' + oo_env + '-openshift-node'] is defined
-
-- name: "Configure instances"
+- name: Configure instances
   hosts: oo_masters_to_config
-  connection: ssh
-  user: root
+  vars:
+    openshift_hostname: "{{ ec2_private_ip_address }}"
+    openshift_public_hostname: "{{ ec2_ip_address }}"
+    # TODO: this should be removed once openshift-sdn packages are available
+    openshift_use_openshift_sdn: False
   vars_files:
-    - vars.yml
+  - vars.yml
   roles:
-    - {
-        role: openshift_master,
-        openshift_node_ips: "{{ hostvars['localhost'].openshift_node_ips | default(['']) }}",
-        openshift_env: "{{ oo_env }}",
-        openshift_public_ip: "{{ ec2_ip_address }}"
-      }
+    - openshift_master
+    #- openshift_sdn_master
     - pods
     - os_env_extras
diff --git a/playbooks/aws/openshift-master/launch.yml b/playbooks/aws/openshift-master/launch.yml
index 3d5a7f579..3d87879a0 100644
--- a/playbooks/aws/openshift-master/launch.yml
+++ b/playbooks/aws/openshift-master/launch.yml
@@ -46,13 +46,16 @@
         tags: "{{ oo_new_inst_tags }}"
 
     - name: Add new instances public IPs to oo_masters_to_config
-      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_masters_to_config"
+      add_host:
+        hostname: "{{ item.0 }}"
+        ansible_ssh_host: "{{ item.1.dns_name }}"
+        groupname: oo_masters_to_config
+        ec2_private_ip_address: "{{ item.1.private_ip }}"
+        ec2_ip_address: "{{ item.1.public_ip }}"
       with_together:
         - oo_new_inst_names
         - ec2.instances
 
-    - debug: var=ec2
-
     - name: Wait for ssh
       wait_for: "port=22 host={{ item.dns_name }}"
       with_items: ec2.instances
diff --git a/playbooks/aws/openshift-master/terminate.yml b/playbooks/aws/openshift-master/terminate.yml
new file mode 100644
index 000000000..fd15cf00f
--- /dev/null
+++ b/playbooks/aws/openshift-master/terminate.yml
@@ -0,0 +1,52 @@
+---
+- name: Populate oo_masters_to_terminate host group if needed
+  hosts: localhost
+  gather_facts: no
+  tasks:
+    - name: Evaluate oo_host_group_exp if it's set
+      add_host: "name={{ item }} groups=oo_masters_to_terminate"
+      with_items: "{{ oo_host_group_exp | default('') }}"
+      when: oo_host_group_exp is defined
+
+- name: Gather facts for instances to terminate
+  hosts: oo_masters_to_terminate
+
+- name: Terminate instances
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars:
+    host_vars: "{{ hostvars
+        | oo_select_keys(groups['oo_masters_to_terminate']) }}"
+  tasks:
+    - name: Terminate instances
+      ec2:
+        state: absent
+        instance_ids: ["{{ item.ec2_id }}"]
+        region: "{{ item.ec2_region }}"
+      ignore_errors: yes
+      register: ec2_term
+      with_items: host_vars
+
+    # Fail if any of the instances failed to terminate with an error other
+    # than 403 Forbidden
+    - fail: msg=Terminating instance {{ item.item.ec2_id }} failed with message {{ item.msg }}
+      when: "item.failed and not item.msg | search(\"error: EC2ResponseError: 403 Forbidden\")"
+      with_items: ec2_term.results
+
+    - name: Stop instance if termination failed
+      ec2:
+        state: stopped
+        instance_ids: ["{{ item.item.ec2_id }}"]
+        region: "{{ item.item.ec2_region }}"
+      register: ec2_stop
+      when: item.failed
+      with_items: ec2_term.results
+
+    - name: Rename stopped instances
+      ec2_tag: resource={{ item.item.item.ec2_id }} region={{ item.item.item.ec2_region }} state=present
+      args:
+        tags:
+          Name: "{{ item.item.item.ec2_tag_Name }}-terminate"
+      with_items: ec2_stop.results
+
diff --git a/playbooks/aws/openshift-master/vars.yml b/playbooks/aws/openshift-master/vars.yml
index fb5f4ea42..c196b2fca 100644
--- a/playbooks/aws/openshift-master/vars.yml
+++ b/playbooks/aws/openshift-master/vars.yml
@@ -1,2 +1,3 @@
 ---
 openshift_debug_level: 4
+openshift_cluster_id: "{{ cluster_id }}"
diff --git a/playbooks/aws/openshift-node/config.yml b/playbooks/aws/openshift-node/config.yml
index 3cf2c58b2..b08ed7571 100644
--- a/playbooks/aws/openshift-node/config.yml
+++ b/playbooks/aws/openshift-node/config.yml
@@ -1,5 +1,5 @@
 ---
-- name: "populate oo_nodes_to_config host group if needed"
+- name: Populate oo_nodes_to_config host group if needed
   hosts: localhost
   gather_facts: no
   tasks:
@@ -7,42 +7,101 @@
     add_host: "name={{ item }} groups=oo_nodes_to_config"
     with_items: "{{ oo_host_group_exp | default('') }}"
     when: oo_host_group_exp is defined
+  - add_host:
+      name: "{{ groups['tag_env-host-type_' ~ cluster_id ~ '-openshift-master'][0] }}"
+      groups: oo_first_master
+    when: oo_host_group_exp is defined
 
-- name: "Gather facts for masters in {{ oo_env }}"
-  hosts: "tag_env-host-type_{{ oo_env }}-openshift-master"
-  connection: ssh
-  user: root
 
-- name: "Set OO sepcific facts on localhost (for later use)"
-  hosts: localhost
-  gather_facts: no
+- name: Gather and set facts for hosts to configure
+  hosts: oo_nodes_to_config
+  roles:
+  - openshift_facts
   tasks:
-    - name: Setting openshift_master_ips fact on localhost
-      set_fact:
-        openshift_master_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type_' + oo_env + '-openshift-master'])
-            | oo_collect(attribute='ansible_default_ipv4.address') }}"
-      when: groups['tag_env-host-type_' + oo_env + '-openshift-master'] is defined
-    - name: Setting openshift_master_public_ips fact on localhost
-      set_fact:
-        openshift_master_public_ips: "{{ hostvars
-            | oo_select_keys(groups['tag_env-host-type-' + oo_env + '-openshift-master'])
-            | oo_collect(attribute='ec2_ip_address') }}"
-      when: groups['tag_env-host-type-' + oo_env + '-openshift-master'] is defined
-
-- name: "Configure instances"
+  # Since the master is registering the nodes before they are configured, we
+  # need to make sure to set the node properties beforehand if we do not want
+  # the defaults
+  - openshift_facts:
+      role: "{{ item.role }}"
+      local_facts: "{{ item.local_facts }}"
+    with_items:
+    - role: common
+      local_facts:
+        hostname: "{{ ec2_private_ip_address }}"
+        public_hostname: "{{ ec2_ip_address }}"
+        # TODO: this should be removed once openshift-sdn packages are available
+        use_openshift_sdn: False
+    - role: node
+      local_facts:
+        external_id: "{{ openshift_node_external_id | default(None) }}"
+        resources_cpu: "{{ openshfit_node_resources_cpu | default(None) }}"
+        resources_memory: "{{ openshfit_node_resources_memory | default(None) }}"
+        pod_cidr: "{{ openshfit_node_pod_cidr | default(None) }}"
+        labels: "{{ openshfit_node_labels | default(None) }}"
+        annotations: "{{ openshfit_node_annotations | default(None) }}"
+
+
+- name: Register nodes
+  hosts: oo_first_master
+  vars:
+    openshift_nodes: "{{ hostvars
+          | oo_select_keys(groups['oo_nodes_to_config']) }}"
+  roles:
+  - openshift_register_nodes
+  tasks:
+  - name: Create local temp directory for syncing certs
+    local_action: command /usr/bin/mktemp -d /tmp/openshift-ansible-XXXXXXX
+    register: mktemp
+
+  - name: Sync master certs to localhost
+    synchronize:
+      mode: pull
+      checksum: yes
+      src: /var/lib/openshift/openshift.local.certificates
+      dest: "{{ mktemp.stdout }}"
+
+
+- name: Configure instances
   hosts: oo_nodes_to_config
-  connection: ssh
-  user: root
   vars_files:
-    - vars.yml
+  - vars.yml
+  vars:
+    openshift_hostname: "{{ ec2_private_ip_address }}"
+    openshift_public_hostname: "{{ ec2_ip_address }}"
+    sync_tmpdir: "{{ hostvars[groups['oo_first_master'][0]].mktemp.stdout }}"
+    cert_parent_rel_path: openshift.local.certificates
+    cert_rel_path: "{{ cert_parent_rel_path }}/node-{{ openshift.common.hostname }}"
+    cert_base_path: /var/lib/openshift
+    cert_parent_path: "{{ cert_base_path }}/{{ cert_parent_rel_path }}"
+    cert_path: "{{ cert_base_path }}/{{ cert_rel_path }}"
+  pre_tasks:
+  - name: Ensure certificate directories exists
+    file:
+      path: "{{ item }}"
+      state: directory
+    with_items:
+    - "{{ cert_path }}"
+    - "{{ cert_parent_path }}/ca"
+
+  # TODO: notify restart openshift-node and/or restart openshift-sdn-node,
+  # possibly test service started time against certificate/config file
+  # timestamps in openshift-node or openshift-sdn-node to trigger notify
+  - name: Sync certs to nodes
+    synchronize:
+      checksum: yes
+      src: "{{ item.src }}"
+      dest: "{{ item.dest }}"
+      owner: no
+      group: no
+    with_items:
+    - src: "{{ sync_tmpdir }}/{{ cert_rel_path }}"
+      dest: "{{ cert_parent_path }}"
+    - src: "{{ sync_tmpdir }}/{{ cert_parent_rel_path }}/ca/cert.crt"
+      dest: "{{ cert_parent_path }}/ca/cert.crt"
+  - local_action: file name={{ sync_tmpdir }} state=absent
+    run_once: true
   roles:
-    - {
-        role: openshift_node,
-        openshift_master_ips: "{{ hostvars['localhost'].openshift_master_ips | default(['']) }}",
-        openshift_master_public_ips: "{{ hostvars['localhost'].openshift_master_public_ips | default(['']) }}",
-        openshift_env: "{{ oo_env }}",
-        openshift_public_ip: "{{ ec2_ip_address }}"
-      }
+    - openshift_node
+    #- openshift_sdn_node
     - os_env_extras
     - os_env_extras_node
diff --git a/playbooks/aws/openshift-node/launch.yml b/playbooks/aws/openshift-node/launch.yml
index 4745fc658..b7ef593e7 100644
--- a/playbooks/aws/openshift-node/launch.yml
+++ b/playbooks/aws/openshift-node/launch.yml
@@ -27,7 +27,9 @@
       register: ec2
 
     - name: Add new instances public IPs to the atomic proxy host group
-      add_host: "hostname={{ item.public_ip }} groupname=new_ec2_instances"
+      add_host:
+        hostname: "{{ item.public_ip }}"
+        groupname: new_ec2_instances"
       with_items: ec2.instances
 
     - name: Add Name and environment tags to instances
@@ -46,13 +48,16 @@
         tags: "{{ oo_new_inst_tags }}"
 
     - name: Add new instances public IPs to oo_nodes_to_config
-      add_host: "hostname={{ item.0 }} ansible_ssh_host={{ item.1.dns_name }} groupname=oo_nodes_to_config"
+      add_host:
+        hostname: "{{ item.0 }}"
+        ansible_ssh_host: "{{ item.1.dns_name }}"
+        groupname: oo_nodes_to_config
+        ec2_private_ip_address: "{{ item.1.private_ip }}"
+        ec2_ip_address: "{{ item.1.public_ip }}"
       with_together:
         - oo_new_inst_names
         - ec2.instances
 
-    - debug: var=ec2
-
     - name: Wait for ssh
       wait_for: "port=22 host={{ item.dns_name }}"
       with_items: ec2.instances
diff --git a/playbooks/aws/openshift-node/terminate.yml b/playbooks/aws/openshift-node/terminate.yml
new file mode 100644
index 000000000..1c0c77eb7
--- /dev/null
+++ b/playbooks/aws/openshift-node/terminate.yml
@@ -0,0 +1,52 @@
+---
+- name: Populate oo_nodes_to_terminate host group if needed
+  hosts: localhost
+  gather_facts: no
+  tasks:
+    - name: Evaluate oo_host_group_exp if it's set
+      add_host: "name={{ item }} groups=oo_nodes_to_terminate"
+      with_items: "{{ oo_host_group_exp | default('') }}"
+      when: oo_host_group_exp is defined
+
+- name: Gather facts for instances to terminate
+  hosts: oo_nodes_to_terminate
+
+- name: Terminate instances
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  vars:
+    host_vars: "{{ hostvars
+        | oo_select_keys(groups['oo_nodes_to_terminate']) }}"
+  tasks:
+    - name: Terminate instances
+      ec2:
+        state: absent
+        instance_ids: ["{{ item.ec2_id }}"]
+        region: "{{ item.ec2_region }}"
+      ignore_errors: yes
+      register: ec2_term
+      with_items: host_vars
+
+    # Fail if any of the instances failed to terminate with an error other
+    # than 403 Forbidden
+    - fail: msg=Terminating instance {{ item.item.ec2_id }} failed with message {{ item.msg }}
+      when: "item.failed and not item.msg | search(\"error: EC2ResponseError: 403 Forbidden\")"
+      with_items: ec2_term.results
+
+    - name: Stop instance if termination failed
+      ec2:
+        state: stopped
+        instance_ids: ["{{ item.item.ec2_id }}"]
+        region: "{{ item.item.ec2_region }}"
+      register: ec2_stop
+      when: item.failed
+      with_items: ec2_term.results
+
+    - name: Rename stopped instances
+      ec2_tag: resource={{ item.item.item.ec2_id }} region={{ item.item.item.ec2_region }} state=present
+      args:
+        tags:
+          Name: "{{ item.item.item.ec2_tag_Name }}-terminate"
+      with_items: ec2_stop.results
+
diff --git a/playbooks/aws/openshift-node/vars.yml b/playbooks/aws/openshift-node/vars.yml
index fb5f4ea42..c196b2fca 100644
--- a/playbooks/aws/openshift-node/vars.yml
+++ b/playbooks/aws/openshift-node/vars.yml
@@ -1,2 +1,3 @@
 ---
 openshift_debug_level: 4
+openshift_cluster_id: "{{ cluster_id }}"