Streamlined networking, OpenShift recovery, Ganesha

41 files changed, 523 insertions, 131 deletions

diff --git a/roles/ands_facts/defaults/main.yml b/roles/ands_facts/defaults/main.yml
index ac61876..fc3fcfd 100644
--- a/roles/ands_facts/defaults/main.yml
+++ b/roles/ands_facts/defaults/main.yml
@@ -1,3 +1,5 @@
+ands_none: "{{ None }}"
+
 ands_configure_heketi: false
 ands_data_device_default_threshold: 10
 
@@ -9,3 +11,21 @@ ands_data_lv: "ands_data"
 ands_data_vg: "{{ ( ansible_lvm['lvs'][ands_data_lv] | default(ands_empty_lv) )['vg'] }}"
 ands_heketi_lv: "ands_heketi"
 ands_heketi_vg: "{{ ( ansible_lvm['lvs'][ands_heketi_lv] | default(ands_empty_lv) )['vg'] }}"
+
+ands_openshift_set_hostname: false
+ands_openshift_set_public_hostname: "{{ (ands_openshift_set_hostname and (ands_openshift_public_network is defined)) | ternary(true, false) }}"
+ands_resolve_public_ip: false
+
+ands_cluster_domain: "{{ ansible_domain }}"
+ands_inner_domain: "{{ ands_cluster_domain }}"
+
+ands_default_ip: "{{ ansible_default_ipv4.address }}"
+ands_openshift_default_ip: "{{ ands_resolve_public_ip | default(false) | ternary(ands_default_ip, ands_none) }}"
+ands_openshift_default_hostname: "{{ (ands_hostname_template is defined) | ternary(ands_hostname_template ~ ands_host_id, ansible_hostname) }}"
+
+ands_inner_lb: false
+ands_inner_lb_id: 254
+ands_inner_lb_hostname: 'ands-lb'
+
+#ands_openshift_inner_interface:
+#ands_openshift_public_interface: 
diff --git a/roles/ands_facts/tasks/find_interface_by_ip.yml b/roles/ands_facts/tasks/find_interface_by_ip.yml
new file mode 100644
index 0000000..ecfa3c3
--- /dev/null
+++ b/roles/ands_facts/tasks/find_interface_by_ip.yml
@@ -0,0 +1,20 @@
+- name: "Looking for interface holding {{ ip }}"
+  set_fact:
+    "{{ var }}": "{{ eth['device'] }}"
+  vars:
+    eth:     "{{ hostvars[inventory_hostname]['ansible_' + item] | default({}) }}"
+    ipv4:    "{{ eth['ipv4'] | default({}) }}"
+    q:       "{{ eth | json_query('ipv4_secondaries[*].address') }}"
+    sec:     "{{ ((q == ands_none) or (q == '')) | ternary([], q) }}"
+    ips:     "{{ sec  | union([ipv4.address]) }}"
+  when: 
+    - eth['type'] is defined
+    - eth['ipv4'] is defined
+    - eth['device'] is defined
+    - eth['type'] == 'ether'
+    - ip in ips
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_interfaces'] }}"
+#  loop_control:
+#    label: "{{ item }}"
+#  no_log: true
diff --git a/roles/ands_facts/tasks/main.yml b/roles/ands_facts/tasks/main.yml
index cf995a0..6b28683 100644
--- a/roles/ands_facts/tasks/main.yml
+++ b/roles/ands_facts/tasks/main.yml
@@ -1,59 +1,14 @@
-- include_vars: dir="vars"
+---
 
-- name: Detect Heketi
-  set_fact: ands_storage_domains="{{ ands_storage_domains | union([ands_heketi_domain]) }}"
-  when: 
-    - ands_configure_heketi
-    - ands_heketi_domain is defined
-    - ansible_lvm.lvs[ands_heketi_lv] is defined
+# The variables accessed trough 'hostvars' should be set as facts
+# Here we set 'ands_storage_servers' and other variables
+- name: "Configuring storage facts"
+  include_tasks: "storage.yml"
 
-- name: Set some facts
-  set_fact:
-    ands_storage_servers: "{{ ands_storage_servers }}"
-
-- name: Set some facts
-  set_fact:
-    ands_data_vg: "{{ ands_data_vg }}"
-  when: ands_data_vg != ""
-
-- name: Set some facts
-  set_fact:
-    ands_data_lv: "{{ ands_data_lv }}"
-  when: ands_data_lv != ""
-
-- name: Set some facts
-  set_fact:
-    ands_heketi_vg: "{{ ands_heketi_vg }}"
-  when: ands_heketi_vg != ""
-
-- name: Set some facts
-  set_fact:
-    ands_heketi_lv: "{{ ands_heketi_lv }}"
-  when: ands_heketi_lv != ""
-
-- name: Set some facts
-  set_fact:
-    ands_data_dev: "/dev/mapper/{{ands_data_vg}}-{{ands_data_lv}}"
-  when: 
-    - ands_data_vg != ""
-    - ands_data_lv != ""
-
-- name: set some facts
-  set_fact:
-    ands_heketi_dev: "/dev/mapper/{{ands_heketi_vg}}-{{ands_heketi_lv}}"
-  when: 
-    - ands_heketi_vg != ""
-    - ands_heketi_lv != ""
+# Here we set 'openshift_hostname', 'openshift_ip' and other variables
+- name: "Configuring network facts"
+  include_tasks: "network.yml"
 
-- name: set some facts
+- name: "Confirm that ands facts are configured"
   set_fact:
-    glusterfs_devices: [ "{{ ands_heketi_dev }}" ]
-  when: 
-    - ands_heketi_vg != ""
-    - ands_heketi_lv != ""
-
-- include_tasks: detect_data_path.yml
-  when: not ands_data_path is defined
-
-#- command: yum-complete-transaction --cleanup-only
-
+    ands_facts_configured: true
diff --git a/roles/ands_facts/tasks/network.yml b/roles/ands_facts/tasks/network.yml
new file mode 100644
index 0000000..1d0248f
--- /dev/null
+++ b/roles/ands_facts/tasks/network.yml
@@ -0,0 +1,49 @@
+- name: Set network facts
+  set_fact:
+    ands_cluster_domain: "{{ ands_cluster_domain }}"
+    ands_cluster_dot_domain: ".{{ ands_cluster_domain }}"
+    ands_inner_domain: "{{ ands_inner_domain }}"
+    ands_inner_dot_domain: "{{ (ands_inner_domain == ands_none) | ternary('', '.' ~ ands_inner_domain) }}"
+    ands_inner_lb_ip: "{{ ands_openshift_network | ipaddr(ands_inner_lb_id) | ipaddr('address') }}"
+    ands_inner_lb_hostname: "{{ ands_inner_lb_hostname }}"
+    ands_openshift_ip: "{{ ands_openshift_network | ipaddr(ands_host_id) | ipaddr('address') }}"
+    ands_openshift_hostname: "{{ ands_openshift_hostname | default(ands_openshift_set_hostname | ternary(ands_openshift_default_hostname, ands_none)) }}"
+    ands_openshift_public_ip: "{{ (ands_openshift_public_network is defined) | ternary( ands_openshift_public_network | ipaddr(ands_host_id) | ipaddr('address'), ands_openshift_default_ip) }}"
+    ands_openshift_public_hostname: "{{ ands_openshift_public_hostname | default(ands_openshift_set_public_hostname | ternary(ands_openshift_default_hostname, ands_none)) }}"
+    ands_storage_ip: "{{ ands_storage_network | default(ands_openshift_network) | ipaddr(ands_host_id) | ipaddr('address') }}"
+    ands_hostname_storage: "ands_storage{{ ands_host_id }}"
+    ands_hostname_openshift: "ands_openshift{{ ands_host_id }}"
+
+- name: Set more network facts
+  set_fact:
+    ands_openshift_public_fqdn: "{{ (ands_openshift_public_hostname == ands_none) | ternary(ands_none, ands_openshift_public_hostname ~ ands_cluster_dot_domain ) }}"
+    ands_openshift_fqdn: "{{ (ands_openshift_hostname == ands_none) | ternary(ands_none, ands_openshift_hostname ~ ands_inner_dot_domain ) }}"
+    ands_openshift_cluster_fqdn: "{{ ands_inner_lb | ternary(ands_inner_lb_hostname ~ ands_inner_dot_domain, ands_openshift_lb) }}"
+
+- name: "Detect inner network interface"
+  include_tasks: "find_interface_by_ip.yml"
+  vars:
+    var: "ands_openshift_inner_interface"
+    ip: "{{ ands_openshift_ip }}"
+  when: 
+    - ands_openshift_inner_interface is not defined
+
+- name: "Detect public network interface"
+  include_tasks: "find_interface_by_ip.yml"
+  vars:
+    var: "ands_openshift_public_interface"
+    ip: "{{ (ands_openshift_public_ip == ands_none) | ternary(ands_default_ip, ands_openshift_public_ip) }}"
+  when: 
+    - ands_openshift_public_interface is not defined
+
+- name: Set ipfailover interface
+  set_fact: 
+    ands_ipfailover_interface: "{{ ands_openshift_public_interface }}"
+  when: ands_ipfailover_interface is not defined
+
+- name: Set ipfailover inner interface
+  set_fact: 
+    ands_ipfailover_inner_interface: "{{ ands_openshift_inner_interface }}"
+  when: ands_ipfailover_inner_interface is not defined
+
+#- debug: msg="{{ hostvars }}"
diff --git a/roles/ands_facts/tasks/storage.yml b/roles/ands_facts/tasks/storage.yml
new file mode 100644
index 0000000..cf995a0
--- /dev/null
+++ b/roles/ands_facts/tasks/storage.yml
@@ -0,0 +1,59 @@
+- include_vars: dir="vars"
+
+- name: Detect Heketi
+  set_fact: ands_storage_domains="{{ ands_storage_domains | union([ands_heketi_domain]) }}"
+  when: 
+    - ands_configure_heketi
+    - ands_heketi_domain is defined
+    - ansible_lvm.lvs[ands_heketi_lv] is defined
+
+- name: Set some facts
+  set_fact:
+    ands_storage_servers: "{{ ands_storage_servers }}"
+
+- name: Set some facts
+  set_fact:
+    ands_data_vg: "{{ ands_data_vg }}"
+  when: ands_data_vg != ""
+
+- name: Set some facts
+  set_fact:
+    ands_data_lv: "{{ ands_data_lv }}"
+  when: ands_data_lv != ""
+
+- name: Set some facts
+  set_fact:
+    ands_heketi_vg: "{{ ands_heketi_vg }}"
+  when: ands_heketi_vg != ""
+
+- name: Set some facts
+  set_fact:
+    ands_heketi_lv: "{{ ands_heketi_lv }}"
+  when: ands_heketi_lv != ""
+
+- name: Set some facts
+  set_fact:
+    ands_data_dev: "/dev/mapper/{{ands_data_vg}}-{{ands_data_lv}}"
+  when: 
+    - ands_data_vg != ""
+    - ands_data_lv != ""
+
+- name: set some facts
+  set_fact:
+    ands_heketi_dev: "/dev/mapper/{{ands_heketi_vg}}-{{ands_heketi_lv}}"
+  when: 
+    - ands_heketi_vg != ""
+    - ands_heketi_lv != ""
+
+- name: set some facts
+  set_fact:
+    glusterfs_devices: [ "{{ ands_heketi_dev }}" ]
+  when: 
+    - ands_heketi_vg != ""
+    - ands_heketi_lv != ""
+
+- include_tasks: detect_data_path.yml
+  when: not ands_data_path is defined
+
+#- command: yum-complete-transaction --cleanup-only
+
diff --git a/roles/ands_network/README b/roles/ands_network/README
new file mode 100644
index 0000000..dfd029a
--- /dev/null
+++ b/roles/ands_network/README
@@ -0,0 +1,8 @@
+ We need to stop keepalived before provisioning OpenShift and scalling the cluster. This will, however, will
+ prevent nodes from communicating with masters. Therefore, we add IP of the first configure master node in 
+ /etc/hosts.
+ 
+ We do the same if for some reason the routing is currently off, but we still want to provision OpenShift 
+ projects. Of course, we don't need to turn off keepalived in this case, just temporarily add an ip of the
+ first master.
+ 
diff --git a/roles/ands_network/tasks/common.yml b/roles/ands_network/tasks/common.yml
new file mode 100644
index 0000000..384029f
--- /dev/null
+++ b/roles/ands_network/tasks/common.yml
@@ -0,0 +1,49 @@
+#- name: Remove obsolte hostnames from /etc/hosts
+#  lineinfile: dest="/etc/hosts" regexp="{{ hostvars[item]['openshift_hostname'] }}" state="absent"
+#  when: 
+#    - hostvars[item]['openshift_hostname'] | default(ands_none) != ands_none
+#    - hostvars[item]['ands_facts_configured'] is defined
+#  with_inventory_hostnames:
+#    - nodes
+#    - new_nodes
+
+
+# This will not work properly unless 'ands_facts' are executed on all nodes.... This is checked by evaluating if 'ands_openshift_fqdn' is defined
+- name: Configure all cluster hostnames in /etc/hosts
+  lineinfile: dest="/etc/hosts" line="{{ ip }} {{ fqdn }} {{ hostname }}" regexp="{{ fqdn }}" state="present"
+  when: 
+    - hostvars[item]['ands_openshift_fqdn'] | default(ands_none) != ands_none
+    - hostvars[item]['ands_facts_configured'] is defined
+  vars:
+    ip: "{{ hostvars[item]['ands_openshift_ip'] }}"
+    fqdn: "{{ hostvars[item]['ands_openshift_fqdn'] }}"
+    hostname: "{{ fqdn.split('.')[0] }}"
+  with_inventory_hostnames:
+    - nodes
+    - new_nodes
+
+- name: Configure all storage ips in /etc/hosts
+  lineinfile: dest="/etc/hosts" line="{{ ip }} {{ hostname }}" regexp="{{ hostname }}" state="present"
+  when: 
+    - hostvars[item]['ands_storage_network'] | default(ands_none) != ands_none
+    - hostvars[item]['ands_facts_configured'] is defined
+  vars:
+    ip: "{{ hostvars[item]['ands_storage_ip'] }}"
+    hostname: "{{ hostvars[item]['ands_hostname_storage'] }}"
+  with_inventory_hostnames:
+    - storage_nodes
+    - new_storage_nodes
+
+
+- name: Provision /etc/hosts to ensure that all masters servers are accessing Master API on loopback device
+  lineinfile: dest="/etc/hosts" line="127.0.0.1 {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="present"
+  when: ('masters' in group_names or 'new_masters' in group_names)
+  register: result
+
+- name: Provision /etc/hosts to ensure that all masters servers are accessing Master API on loopback device
+  lineinfile: dest="/etc/hosts" line="{{ ands_inner_lb_ip }} {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="present"
+  when: (result | skipped) and (ands_inner_lb | default(false))
+
+- name: Register openshift_dns_ip in /etc/hosts
+  lineinfile: dest="/etc/hosts" line="{{ openshift_dns_ip }} openshift_dns_ip" regexp="openshift_dns_ip$" state="present"
+
diff --git a/roles/ands_network/tasks/install_post.yml b/roles/ands_network/tasks/install_post.yml
new file mode 100644
index 0000000..0bfef34
--- /dev/null
+++ b/roles/ands_network/tasks/install_post.yml
@@ -0,0 +1,9 @@
+- name: Start keepalived
+  service: name=keepalived state=started enabled=yes
+  when: ('masters' in group_names) or ('new_masters' in group_names)
+
+- name: Provision /etc/hosts to ensure that all hosts accessing masters servers appropriately
+  lineinfile: dest="/etc/hosts" line="{{ ands_inner_lb_ip | default('') }} {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="{{ state }}"
+  when: ('masters' not in group_names and 'new_masters' not in group_names)
+  vars:
+    state: "{{ ands_inner_lb | default(false) | ternary('present', 'absent') }}"
diff --git a/roles/ands_network/tasks/install_pre.yml b/roles/ands_network/tasks/install_pre.yml
new file mode 100644
index 0000000..f555d1b
--- /dev/null
+++ b/roles/ands_network/tasks/install_pre.yml
@@ -0,0 +1,15 @@
+- name: Temporary provision /etc/hosts with Masters IP.
+  lineinfile: dest="/etc/hosts" line="{{ ands_openshift_network | ipaddr(node_id) | ipaddr('address') }} {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="present"
+  when: ('masters' not in group_names)
+  vars:
+    node_id: "{{ hostvars[groups['masters'][0]]['ands_host_id'] }}"
+
+- name: Check if keepalived is installed
+  stat: path="/etc/sysconfig/keepalived"
+  register: keepalived_result
+
+- name: Stop keepalived
+  service: name=keepalived state=stopped
+  when: 
+    - keepalived_result.stat.exists
+    - ('masters' in group_names) or ('new_masters' in group_names)
diff --git a/roles/ands_network/tasks/main.yml b/roles/ands_network/tasks/main.yml
new file mode 100644
index 0000000..0bc913a
--- /dev/null
+++ b/roles/ands_network/tasks/main.yml
@@ -0,0 +1,3 @@
+---
+- name: "Configuring network"
+  include_tasks: "{{ action | default('common') }}.yml"
diff --git a/roles/ands_network/tasks/maintain.yml b/roles/ands_network/tasks/maintain.yml
new file mode 100644
index 0000000..a7af597
--- /dev/null
+++ b/roles/ands_network/tasks/maintain.yml
@@ -0,0 +1,9 @@
+- name: Ensure keepalived is running on master nodes
+  service: name=keepalived state=started enabled=yes
+  when: ('masters' in group_names)
+
+- name: Provision /etc/hosts to ensure that all hosts accessing masters servers appropriately
+  lineinfile: dest="/etc/hosts" line="{{ ands_inner_lb_ip | default('') }} {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="{{ state }}"
+  when: ('masters' not in group_names and 'new_masters' not in group_names)
+  vars:
+    state: "{{ ands_inner_lb | default(false) | ternary('present', 'absent') }}"
diff --git a/roles/ands_openshift/defaults/main.yml b/roles/ands_openshift/defaults/main.yml
index b97b584..d279345 100644
--- a/roles/ands_openshift/defaults/main.yml
+++ b/roles/ands_openshift/defaults/main.yml
@@ -1,4 +1,4 @@
-openshift_common_subroles: "{{ [ 'hostnames', 'users', 'security', 'storage' ] }}"
+openshift_common_subroles: "{{ [ 'users', 'security', 'storage' ] }}"
 openshift_heketi_subroles: "{{ [ 'ssh', 'heketi' ] }}"
 openshift_all_subroles: "{{ ands_configure_heketi | default(False) | ternary(openshift_common_subroles + openshift_heketi_subroles, openshift_common_subroles) }}"
 
diff --git a/roles/ands_openshift/tasks/hostnames.yml b/roles/ands_openshift/tasks/hostnames.yml
deleted file mode 100644
index e489a8c..0000000
--- a/roles/ands_openshift/tasks/hostnames.yml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-#- name: Remove obsolte hostnames from /etc/hosts
-#  lineinfile: dest="/etc/hosts" regexp="{{ hostvars[item]['openshift_hostname'] }}" state="absent"
-#  with_inventory_hostnames:
-#    - nodes
-
-
-- name: Configure all cluster hostnames in /etc/hosts
-  lineinfile: dest="/etc/hosts" line="{{ hostvars[item]['openshift_ip'] }} {{ hostvars[item]['openshift_public_hostname'] }} {{ hostvars[item]['openshift_hostname'] }}" regexp="{{ hostvars[item]['openshift_hostname'] }}" state="present"
-  with_inventory_hostnames:
-    - nodes
-
-- name: Provision /etc/hosts to ensure that all masters servers are accessing Master API on loopback device
-  lineinfile: dest="/etc/hosts" line="127.0.0.1 {{ openshift_master_cluster_hostname }}" regexp=".*{{ openshift_master_cluster_hostname }}$" state="present"
-  when: "'masters' in group_names"
diff --git a/roles/ands_storage/tasks/detect_device.yml b/roles/ands_storage/tasks/detect_device.yml
index 0fb9764..3467371 100644
--- a/roles/ands_storage/tasks/detect_device.yml
+++ b/roles/ands_storage/tasks/detect_device.yml
@@ -1,3 +1,8 @@
+#- name: find if ands data is already mounted
+#  set_fact: ands_data_is_mounted=true
+#  with_items: "{{ ansible_mounts }}"
+#  when: item.mount == ands_data_path 
+
 - name: find large block devices
   set_fact: ands_data_device="/dev/{{ item.key }}"
 #  debug: msg="{{ item.key }} - {{ (item.value.sectors | int) * (item.value.sectorsize | int) / 1024 / 1024 / 1024 }} GB"
diff --git a/roles/ands_storage/tasks/main.yml b/roles/ands_storage/tasks/main.yml
index 9318f88..43d4692 100644
--- a/roles/ands_storage/tasks/main.yml
+++ b/roles/ands_storage/tasks/main.yml
@@ -10,11 +10,15 @@
 
 - name: Create Ands VG
   lvg: vg="{{ ands_data_vg }}" pvs="{{ ands_data_device }}"
-  when: ands_data_device is defined
+  when: 
+    - ands_data_device is defined
+    - ansible_lvm.vgs[ands_data_vg] is not defined
 
 - name: Create Heketi VG
   lvg: vg="{{ ands_heketi_vg }}" pvs="{{ ands_heketi_device }}"
-  when: ands_heketi_device is defined
+  when: 
+    - ands_heketi_device is defined
+    - ansible_lvm.vgs[ands_heketi_vg] is not defined
 
 - name: Check if Heketi Volume already exists
   stat: path="/dev/{{ ands_heketi_vg }}/{{ ands_heketi_lv }}"
diff --git a/roles/ands_vagrant_vm/templates/Vagrantfile.j2 b/roles/ands_vagrant_vm/templates/Vagrantfile.j2
index b044e2e..386ba85 100644
--- a/roles/ands_vagrant_vm/templates/Vagrantfile.j2
+++ b/roles/ands_vagrant_vm/templates/Vagrantfile.j2
@@ -12,7 +12,7 @@ Vagrant.configure("2") do |config|
   (1..{{ vagrant_hosts }}).each do |i|
     config.vm.define "{{ vagrant_hostname_template }}#{i}" do |node|
         node.vm.network "public_network", nm_controlled: "yes", bridge: "br0", mac: "080027{{ macid  }}02#{i}", ip: "{{ public_net }}.#{i}", type: "dhcp"
-        node.vm.network "private_network", nm_controlled: "yes",  mac: "080027{{ macid }}12#{i}", ip: "{{ storage_net }}.#{i}", name: "vboxnet0", type: "static"
+        node.vm.network "private_network", nm_controlled: "yes",  mac: "080027{{ macid }}12#{i}", ip: "{{ net }}.#{i}", name: "vboxnet0", type: "static"
         node.vm.box = "centos/7"
         node.disksize.size = "80 GB"
         node.vm.hostname = "{{ vagrant_hostname_template }}#{i}.ipe.kit.edu"
@@ -26,8 +26,9 @@ Vagrant.configure("2") do |config|
         node.vm.provision "shell", run: "always", inline: "( ip addr show dev eth2 | grep {{ netid }}.#{i} ) || ip addr add 192.168.{{ netid }}.#{i}/24 dev eth2"
         node.vm.provision "shell", run: "always", inline: "chmod +r /etc/sysconfig/network-scripts/ifcfg-eth*"
         node.vm.provision "shell", run: "always", inline: "chcon --reference /etc/sysconfig/network-scripts/ifcfg-eth0 /etc/sysconfig/network-scripts/ifcfg-eth*"
-
-        node.vm.provision "shell", run: "always", inline: "ip route del default dev eth0"
+#        node.vm.provision "shell", run: "always", inline: "nmcli con down 'System eth0'; nmcli con up 'System eth0'"
+        node.vm.provision "shell", run: "always", inline: "ip route del default dev eth0 &> /dev/null ; error=$?"
+        node.vm.provision "shell", run: "always", inline: "DEVICE_IFACE=eth1 /etc/NetworkManager/dispatcher.d/99-origin-dns.sh eth1 up &> /dev/null; error=$?"
         
         node.vm.provision "shell" do |s|
             ssh_pub_key = File.readlines("authorized_keys").first.strip
diff --git a/roles/ands_vagrant_vmconf/tasks/main.yml b/roles/ands_vagrant_vmconf/tasks/main.yml
index f52a52d..b130aa4 100644
--- a/roles/ands_vagrant_vmconf/tasks/main.yml
+++ b/roles/ands_vagrant_vmconf/tasks/main.yml
@@ -23,6 +23,4 @@
 # We just need networkmanager running   
 # - name: Bypass absent NM
 #   copy: remote_src="yes" src="/etc/resolv.conf" dest="/etc/origin/node/resolv.conf" 
-   
- - name: Update CentOS
-   yum: name=* state=latest update_cache=yes
+  
diff --git a/roles/common/default/main.yml b/roles/common/default/main.yml
new file mode 100644
index 0000000..d355d15
--- /dev/null
+++ b/roles/common/default/main.yml
@@ -0,0 +1 @@
+os_update: "{{ ands_update | default(false) }}"
\ No newline at end of file
diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml
index 9bd820a..fdd7246 100644
--- a/roles/common/tasks/main.yml
+++ b/roles/common/tasks/main.yml
@@ -14,6 +14,7 @@
 # Seems we need iptables-services at least temporary...
 - name: Ensure all required packages are installed
   package: name={{item}} state=present
+  register: result
   with_items:
     - mc
     - bzr
@@ -28,6 +29,12 @@
     - PyYAML
     - python-rhsm-certificates
     - glusterfs-fuse
+    - telnet
+
+# We always update on first install and if requested
+- name: Update CentOS
+  yum: name=* state=latest update_cache=yes
+  when: (result | changed) or (os_update | default(false))
 
 #- name: Add NodeJS required by a few used Ansible extensions
 #  package: name={{item}} state=present
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index a7bd700..0d040a9 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -27,14 +27,16 @@
   lvol: vg="{{ ansible_lvm['lvs'][docker_lv]['vg'] }}" lv="docker_lv" size="{{ docker_volume_size }}"
   when: docker_volume_size is defined
 
-- name: Limit size of container log files
-  ghetto_json:
-    path: "/etc/docker/daemon.json"
-    log-driver: "json-file"
-    log-opts.max-size: "{{ docker_max_log_size }}" 
-    log-opts.max-file: "{{ docker_max_log_files }}"
-  notify:
-    - restart docker
+# By default there is systemd driver installed. It is removed during OpenShift installation, but is still there during prepare stage
+# The parameters to docker can be set trough OpenShift and currently are moved there.
+#- name: Limit size of container log files
+#  ghetto_json:
+#    path: "/etc/docker/daemon.json"
+#    log-driver: "json-file"
+#    log-opts.max-size: "{{ docker_max_log_size }}" 
+#    log-opts.max-file: "{{ docker_max_log_files }}"
+#  notify:
+#    - restart docker
 
 - name: start docker
   service: name="docker" enabled=yes state=started
diff --git a/roles/docker/tasks/storage.yml b/roles/docker/tasks/storage.yml
index 595979c..5a5c858 100644
--- a/roles/docker/tasks/storage.yml
+++ b/roles/docker/tasks/storage.yml
@@ -20,7 +20,7 @@
 - set_fact: docker_storage_config="VG={{ docker_storage_vg }} AUTO_EXTEND_POOL=true"
 
 - set_fact: docker_storage_config="{{ docker_storage_config }} DEVS={{ docker_storage_device }}"
-  when: ( docker_storage_device is defined ) and ( not ansible_lvm.vgs.{{ docker_storage_vg }} is defined )
+  when: ( docker_storage_device is defined ) and ( ansible_lvm.vgs[docker_storage_vg] is not defined )
   
 - name: stop docker
   service: name="docker" state="stopped"
diff --git a/roles/ganesha/files/ganesha.conf b/roles/ganesha/files/ganesha.conf
new file mode 100644
index 0000000..2bfc114
--- /dev/null
+++ b/roles/ganesha/files/ganesha.conf
@@ -0,0 +1 @@
+%include "/etc/ganesha/exports/export.openshift.conf"
diff --git a/roles/ganesha/tasks/main.yml b/roles/ganesha/tasks/main.yml
new file mode 100644
index 0000000..032631b
--- /dev/null
+++ b/roles/ganesha/tasks/main.yml
@@ -0,0 +1,30 @@
+- name: Ensure GlusterFS repositories are present
+  yum: name="centos-release-gluster{{ glusterfs_version }}" state=present
+
+- name: Ensure Ganesha is installed
+  yum: name={{item}} state=present
+  with_items:
+    - nfs-ganesha-gluster
+    - nfs-ganesha
+
+- name: Change logdir group to prevent selinux problems
+  file: dest="/var/log/ganesha" owner="ganesha" group="root" mode="0775" state="directory"
+
+- name: Copy default Ganesha configuration
+  copy: src="ganesha.conf" dest="/etc/ganesha/ganesha.conf" owner="root" group="root" mode="0644"
+
+- name: Configure Ganesha HA
+  template: src="ganesha-ha.conf.j2" dest="/etc/ganesha/ganesha-ha.conf" owner=root group=root mode="0644"
+
+- name: Configure firewalld
+  firewalld: service="{{ item }}" state="enabled" permanent="true" immediate="true"
+  with_items:
+    - nfs
+#    - mountd
+#    - rpc-bind
+
+- name: Reload firewalld rules
+  shell: firewall-cmd --reload
+
+- name: Enable and start ganesha service
+  service: name="nfs-ganesha" state=started enabled=yes
diff --git a/roles/ganesha/templates/ganesha-ha.conf.j2 b/roles/ganesha/templates/ganesha-ha.conf.j2
new file mode 100644
index 0000000..bdb2e0c
--- /dev/null
+++ b/roles/ganesha/templates/ganesha-ha.conf.j2
@@ -0,0 +1,12 @@
+{% set members = groups['masters'] | union(groups['new_masters'] | default([])) | map('extract', hostvars, 'ands_hostname_storage') | list %}
+{% set vips = ands_ipfailover_vips | default([]) %}
+{% set n_vips = vips | length %}
+{% if n_vips > 0 %}
+{%    set nodes = members[0:n_vips] %}
+HA_NAME="openshift_nfs"
+#HA_VOL_SERVER="{{ hostvars[groups['masters'][0]]['ands_hostname_storage'] }}"
+HA_CLUSTER_NODES="{{ nodes | join(',') }}"
+{%   for node in nodes %}
+VIP_{{ node }}="{{ vips[loop.index - 1] }}"
+{%   endfor %}
+{% endif %}
diff --git a/roles/glusterfs/defaults/main.yml b/roles/glusterfs/defaults/main.yml
index 9587a9b..700838d 100644
--- a/roles/glusterfs/defaults/main.yml
+++ b/roles/glusterfs/defaults/main.yml
@@ -1,5 +1,5 @@
 ---
-glusterfs_version: 39
+glusterfs_version: 312
 glusterfs_transport: rdma
 
 glusterfs_network: "{{ ands_storage_network }}"
diff --git a/roles/glusterfs/tasks/common.yml b/roles/glusterfs/tasks/common.yml
index 5e8e3b6..67fb815 100644
--- a/roles/glusterfs/tasks/common.yml
+++ b/roles/glusterfs/tasks/common.yml
@@ -8,6 +8,7 @@
     - glusterfs-cli
     - glusterfs-fuse
     - glusterfs-rdma
+    - heketi-client
     - libsemanage-python
             
 - name: Allow fuse in SELinux configuration
diff --git a/roles/glusterfs/tasks/create_domain.yml b/roles/glusterfs/tasks/create_domain.yml
index 8f8042b..76623f2 100644
--- a/roles/glusterfs/tasks/create_domain.yml
+++ b/roles/glusterfs/tasks/create_domain.yml
@@ -1,8 +1,16 @@
 ---
+- name: Get list of existing gluster volumes
+  shell: "gluster volume info"
+  changed_when: false
+  register: gv_results
+
+
 - name: Configure volumes
   include_tasks: create_volume.yml
   with_dict: "{{ domain.volumes }}"
+  when: volume_string not in gv_results.stdout_lines
   vars:
+    volume_string: "Volume Name: {{ volume.key }}"
     domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}"
   loop_control:
     loop_var: volume
diff --git a/roles/glusterfs/tasks/ganesha.yml b/roles/glusterfs/tasks/ganesha.yml
new file mode 100644
index 0000000..61d151a
--- /dev/null
+++ b/roles/glusterfs/tasks/ganesha.yml
@@ -0,0 +1,6 @@
+- name: Create /etc/ganesha/exports
+  file: dest="/etc/ganesha/exports" owner="root" group="root" mode="0755" state="directory"
+
+- name: Configure Ganesha NFS exports
+  template: src="export.openshift.conf.j2" dest="/etc/ganesha/exports/export.openshift.conf" owner=root group=root mode="0644"
+
diff --git a/roles/glusterfs/tasks/install.yml b/roles/glusterfs/tasks/install.yml
new file mode 100644
index 0000000..d7ee766
--- /dev/null
+++ b/roles/glusterfs/tasks/install.yml
@@ -0,0 +1,34 @@
+---
+- name: Install GlusterFS Common Software
+  include_tasks: common.yml
+  when:
+    - "'software' in glusterfs_subroles"
+
+- name: Install GlusterFS client
+  include_tasks: setup-client.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' not in group_names"
+
+- name: Install GlusterFS OpenShift Server
+  include_tasks: setup-openshift-server.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' in group_names"
+    - "'glusterfs' in group_names"
+
+- name: Install GlusterFS External Server
+  include_tasks: setup-external-server.yml
+  when: 
+    - "'software' in glusterfs_subroles"
+    - "'ands_storage_servers' in group_names"
+    - "'glusterfs' not in group_names"
+
+- name: Configure gluster peers (on first host)
+  shell: gluster peer probe {{item}}
+  run_once: true
+  with_items: "{{ glusterfs_servers }}"
+
+- include_tasks: volumes.yml
+  when:
+    - "'volumes' in glusterfs_subroles"
diff --git a/roles/glusterfs/tasks/iterate_domains.yml b/roles/glusterfs/tasks/iterate_domains.yml
new file mode 100644
index 0000000..e061652
--- /dev/null
+++ b/roles/glusterfs/tasks/iterate_domains.yml
@@ -0,0 +1,7 @@
+- name: Process all storage domains
+  include_tasks: "iterate_volumes.yml"
+  run_once: true
+  delegate_to: "{{ groups[domain.servers][0] }}"
+  with_items: "{{ glusterfs_domains }}"
+  loop_control:
+    loop_var: domain  
diff --git a/roles/glusterfs/tasks/iterate_volumes.yml b/roles/glusterfs/tasks/iterate_volumes.yml
new file mode 100644
index 0000000..8f61116
--- /dev/null
+++ b/roles/glusterfs/tasks/iterate_volumes.yml
@@ -0,0 +1,12 @@
+---
+- name: Iterate volumes
+  include_tasks: "{{ action }}.yml"
+  with_dict: "{{ domain.volumes }}"
+  vars:
+    name: "{{ volume.key }}"
+    path: "{{ volume.value.mount }}"
+    server_group: "{{ domain.servers }}"
+    domain_servers: "{{ groups[domain.servers] | map('extract', hostvars, 'ands_storage_hostname') | list }}"
+  when: volume.value.mount is defined
+  loop_control:
+    loop_var: volume
diff --git a/roles/glusterfs/tasks/main.yml b/roles/glusterfs/tasks/main.yml
index d7ee766..a02c1a1 100644
--- a/roles/glusterfs/tasks/main.yml
+++ b/roles/glusterfs/tasks/main.yml
@@ -1,34 +1,3 @@
 ---
-- name: Install GlusterFS Common Software
-  include_tasks: common.yml
-  when:
-    - "'software' in glusterfs_subroles"
-
-- name: Install GlusterFS client
-  include_tasks: setup-client.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' not in group_names"
-
-- name: Install GlusterFS OpenShift Server
-  include_tasks: setup-openshift-server.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' in group_names"
-    - "'glusterfs' in group_names"
-
-- name: Install GlusterFS External Server
-  include_tasks: setup-external-server.yml
-  when: 
-    - "'software' in glusterfs_subroles"
-    - "'ands_storage_servers' in group_names"
-    - "'glusterfs' not in group_names"
-
-- name: Configure gluster peers (on first host)
-  shell: gluster peer probe {{item}}
-  run_once: true
-  with_items: "{{ glusterfs_servers }}"
-
-- include_tasks: volumes.yml
-  when:
-    - "'volumes' in glusterfs_subroles"
+- name: "Configuring Gluster storage subsystem"
+  include_tasks: "{{ action | default('install') }}.yml"
diff --git a/roles/glusterfs/tasks/migrate.yml b/roles/glusterfs/tasks/migrate.yml
new file mode 100644
index 0000000..3ef7917
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate.yml
@@ -0,0 +1,34 @@
+- debug: msg="Migrating"
+
+#- name: Detect if source host {{ glusterfs_migrate_from }} is failed or running
+#  wait_for: host={{ glusterfs_migrate_from }} port=24007 timeout=1
+#  register: srcres
+#  changed_when: srcres | failed
+#  failed_when: false
+
+#- set_fact: 
+#    glusterfs_migrate_failed: "{{ srcres | changed }}"
+
+- set_fact: 
+    glusterfs_migrate_failed: "1"
+
+- name: Analyze current configuration
+  include_tasks: iterate_domains.yml
+  vars:
+    action: "migrate_volume"
+    migrate_action: "migrate_inform"
+    migrate_from: "{{ glusterfs_migrate_from }}"
+    migrate_to: "{{ glusterfs_migrate_to }}"
+    migrate_failed: "{{ glusterfs_migrate_failed }}"
+
+- name: Exterminate mankind
+  pause: prompt='Please confirm if you want to proceed. Press Ctrl+c and then "a" to abort'
+
+- name: Analyze current configuration
+  include_tasks: iterate_domains.yml
+  vars:
+    action: "migrate_volume"
+    migrate_action: "{{ migrate_failed | ternary('migrate_failed_brick', 'migrate_live_brick') }}"
+    migrate_from: "{{ glusterfs_migrate_from }}"
+    migrate_to: "{{ glusterfs_migrate_to }}"
+    migrate_failed: "{{ glusterfs_migrate_failed }}"
diff --git a/roles/glusterfs/tasks/migrate_failed_brick.yml b/roles/glusterfs/tasks/migrate_failed_brick.yml
new file mode 100644
index 0000000..3490c82
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_failed_brick.yml
@@ -0,0 +1,10 @@
+- name: "Volume {{ vol }} - Migrating {{ src }} to {{ dst }}"
+  command: "gluster volume replace-brick {{ vol }} {{ src }} {{ dst }} commit force"
+
+# This is only working within containers with 'oc rsh'. But actually there are auto-healing daemons running.
+# So we probably find just migrating
+#- name: "Healing {{ vol }}"
+#  command: "gluster volume heal {{ vol }} full"
+
+#- name: "Rebalancing {{ vol }}"
+#  command: "gluster volume rebalance {{ vol }} fix-layout start"
diff --git a/roles/glusterfs/tasks/migrate_inform.yml b/roles/glusterfs/tasks/migrate_inform.yml
new file mode 100644
index 0000000..912d359
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_inform.yml
@@ -0,0 +1 @@
+- warn: msg="Volume {{ vol }} - Migrating {{ src }} to {{ dst }}"
diff --git a/roles/glusterfs/tasks/migrate_volume.yml b/roles/glusterfs/tasks/migrate_volume.yml
new file mode 100644
index 0000000..f9edeac
--- /dev/null
+++ b/roles/glusterfs/tasks/migrate_volume.yml
@@ -0,0 +1,17 @@
+- name: "Analyze bricks of {{ volume.key }}"
+  shell: |
+    gluster volume info '{{ volume.key }}' | grep -P 'Brick\d+:' | awk '{ print $2 }'
+  register: gvires
+
+- name: "Execute configured {{ migrate_action }} on volume {{ volume.key }} with bricks {{ src }} and {{ dst }}"
+  include_tasks: "{{ migrate_action }}.yml"
+  vars:
+    bricks: "{{ gvires.stdout_lines | list }}"
+    servers: "{{ bricks | map('regex_replace', ':.*$', '') | list }}"
+    brick: "{{ servers.index(migrate_from) }}"
+    src: "{{ bricks[brick | int] | default('') }}"
+    dst: "{{ bricks[brick | int] | default('') | regex_replace('^' ~ migrate_from, migrate_to) }}"
+    vol: "{{ volume.key }}"
+  when:
+    - migrate_from in servers
+    - brick is defined
diff --git a/roles/glusterfs/tasks/volumes.yml b/roles/glusterfs/tasks/volumes.yml
index c4d49ac..1a85378 100644
--- a/roles/glusterfs/tasks/volumes.yml
+++ b/roles/glusterfs/tasks/volumes.yml
@@ -8,7 +8,7 @@
 
 - name: Mount volume domains
   include_tasks: mount_domain.yml
-  when: ( domain.clients | default("---") ) in group_names
+  when: domain.clients | default([]) | intersect(group_names) | length > 0
   with_items: "{{ glusterfs_domains }}"
   loop_control:
     loop_var: domain  
diff --git a/roles/glusterfs/templates/export.openshift.conf.j2 b/roles/glusterfs/templates/export.openshift.conf.j2
new file mode 100644
index 0000000..b2c547f
--- /dev/null
+++ b/roles/glusterfs/templates/export.openshift.conf.j2
@@ -0,0 +1,44 @@
+{% set i = 0 %}
+{% for domain in glusterfs_domains %}
+{%   for name, vol in domain.volumes.iteritems() %}
+{%      if vol.nfs_clients is defined %}
+{%        set nfs = vol.nfs_clients %}
+{%        set i = i + 1 %}
+EXPORT {
+      Export_Id = {{ i }};
+      Path = "/{{ name }}";
+      FSAL {
+            name = GLUSTER;
+            hostname = "localhost";
+            volume = "{{ name }}";
+      }
+      Access_type = RW;
+      Disable_ACL = true;
+      Squash = "No_root_squash";
+      Pseudo = "/{{ name }}";
+      Protocols = "3", "4" ;
+      Transports = "UDP","TCP";
+      SecType = "sys";
+
+{%        if nfs.rw is defined %}
+{%          for net in nfs.rw %}
+      CLIENT {
+            clients = {{ net }};
+            Access_type = RW;
+      }
+{%          endfor %}
+{%        endif  %}
+
+{%        if nfs.ro is defined %}
+{%          for net in nfs.ro %}
+      CLIENT {
+            clients = {{ net }};
+            Access_type = RO;
+      }
+{%          endfor %}
+{%        endif  %}
+}
+
+{%     endif %}
+{%   endfor %}
+{% endfor %}
diff --git a/roles/keepalived/defaults/main.yml b/roles/keepalived/defaults/main.yml
index a7087b0..3302b85 100644
--- a/roles/keepalived/defaults/main.yml
+++ b/roles/keepalived/defaults/main.yml
@@ -1,6 +1,9 @@
 ---
-keepalived_vips: "{{ ands_ipfailover_vips | default([]) }}"
-keepalived_iface: "{{ ands_ipfailover_interface | default('eth0') }}"
+keepalived_vrrp:
+  - vips: "{{ ands_ipfailover_vips | default([]) }}"
+    iface: "{{ ands_ipfailover_interface | default('eth0') }}"
+  - vips: "{{ ands_inner_lb | ternary([ands_inner_lb_ip], []) }}"
+    iface: "{{ ands_ipfailover_inner_interface }}"
 
 keepalived_master_prio: 80
 keepalived_backup_prio: 20
diff --git a/roles/keepalived/tasks/main.yml b/roles/keepalived/tasks/main.yml
index adedcdc..08835b8 100644
--- a/roles/keepalived/tasks/main.yml
+++ b/roles/keepalived/tasks/main.yml
@@ -20,3 +20,4 @@
 - name: Start keepalived
   service: name=keepalived state=started enabled=yes
   tags: keepalived
+
diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2
index 8d9a580..6df5eab 100644
--- a/roles/keepalived/templates/keepalived.conf.j2
+++ b/roles/keepalived/templates/keepalived.conf.j2
@@ -6,20 +6,22 @@ vrrp_script track {
     interval {{ keepalived_check_interval }}
 }
 
-{% for vips in keepalived_vips  %}
-{% set id = (  vips | ipaddr('address') | regex_replace('^.*\.', '') ) %}
+{% for vrrp in keepalived_vrrp  %}
+{%   set vrrp_id = loop.index %}
+{%   for vips in vrrp.vips  %}
+{%     set id = (  vips | ipaddr('address') | regex_replace('^.*\.', '') ) %}
 
-vrrp_instance VI_{{ loop.index }} {
+vrrp_instance VI_{{vrrp_id}}_{{ loop.index }} {
 
-    virtual_router_id {{ id }}
+    virtual_router_id {{ (vrrp_id - 1) * 32 + (loop.index) }}
 
     state {{ (( ( loop.index - 1) % (keepalived_num_nodes | int) ) == (keepalived_node_id | int) ) | ternary('MASTER', 'BACKUP') }}
     state {{ (( ( loop.index - 1) % (keepalived_num_nodes | int) ) == (keepalived_node_id | int) ) | ternary(keepalived_master_prio, keepalived_backup_prio) }}
 
-    interface {{ keepalived_iface }}
+    interface {{ vrrp.iface }}
 
     virtual_ipaddress {
-        {{ vips }} dev {{ keepalived_iface }} 
+        {{ vips }} dev {{ vrrp.iface }} 
     }
     
     advert_int 1
@@ -33,4 +35,5 @@ vrrp_instance VI_{{ loop.index }} {
         track
     }
 }
+{%   endfor %}
 {% endfor %}