update vars to allow scaling of components (#9)

11 files changed, 173 insertions, 23 deletions

diff --git a/roles/openshift_metrics/defaults/main.yaml b/roles/openshift_metrics/defaults/main.yaml
index ae24e1972..c27943220 100644
--- a/roles/openshift_metrics/defaults/main.yaml
+++ b/roles/openshift_metrics/defaults/main.yaml
@@ -1,9 +1,11 @@
 ---
+openshift_metrics_start_cluster: True
 openshift_metrics_install_metrics: True
 openshift_metrics_image_prefix: docker.io/openshift/origin-
 openshift_metrics_image_version: latest
 openshift_metrics_master_url: https://kubernetes.default.svc.cluster.local
 openshift_metrics_project: openshift-infra
+openshift_metrics_certs_dir: "{{ openshift.common.config_base }}/master/metrics"
 openshift_metrics_startup_timeout: 500
 openshift_metrics_certs_dir: "{{ openshift.common.config_base }}/master/metrics"
 
diff --git a/roles/openshift_metrics/tasks/install_hawkular.yaml b/roles/openshift_metrics/tasks/install_hawkular.yaml
index 6e503c8c1..1acc8948d 100644
--- a/roles/openshift_metrics/tasks/install_hawkular.yaml
+++ b/roles/openshift_metrics/tasks/install_hawkular.yaml
@@ -3,6 +3,7 @@
   template:
     src: hawkular_metrics_rc.j2
     dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_rc.yaml"
+
 - name: generate hawkular-cassandra replication controllers
   template:
     src: hawkular_cassandra_rc.j2
@@ -45,6 +46,7 @@
 - name: read hawkular-metrics route destination ca certificate
   slurp: src={{ openshift_metrics_certs_dir }}/ca.crt
   register: metrics_route_dest_ca_cert
+
 - name: generate the hawkular-metrics route
   template:
     src: route.j2
diff --git a/roles/openshift_metrics/tasks/install_metrics.yaml b/roles/openshift_metrics/tasks/install_metrics.yaml
index db023e6a2..a6a094a83 100644
--- a/roles/openshift_metrics/tasks/install_metrics.yaml
+++ b/roles/openshift_metrics/tasks/install_metrics.yaml
@@ -1,4 +1,15 @@
 ---
+- name: check that hawkular_metrics_hostname is set
+  fail: msg='the openshift_metrics_hawkular_metrics_hostname variable is required'
+  when: openshift_metrics_hawkular_metrics_hostname is not defined
+
+- name: check the value of openshift_metrics_cassandra_storage_type
+  fail:
+    msg: >
+      openshift_metrics_cassandra_storage_type ({{ openshift_metrics_cassandra_storage_type }})
+      is invalid, must be one of: emptydir, pv, dynamic
+  when: openshift_metrics_cassandra_storage_type not in openshift_metrics_cassandra_storage_types
+
 - name: Install Metrics
   include: "{{ role_path }}/tasks/install_{{ include_file }}.yaml"
   with_items:
@@ -7,3 +18,17 @@
     - hawkular
   loop_control:
     loop_var: include_file
+
+- name: create objects
+  command: >
+    {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }}
+    --config={{ mktemp.stdout }}/admin.kubeconfig
+    apply -f {{ item }}
+  with_fileglob:
+  - "{{ mktemp.stdout }}/templates/*.yaml"
+
+- name: Scaling up cluster
+  include: start_metrics.yaml
+  tags: openshift_metrics_start_cluster
+  when:
+  - openshift_metrics_start_cluster | default(true) | bool
diff --git a/roles/openshift_metrics/tasks/main.yaml b/roles/openshift_metrics/tasks/main.yaml
index 74abd120f..e8c74b8dc 100644
--- a/roles/openshift_metrics/tasks/main.yaml
+++ b/roles/openshift_metrics/tasks/main.yaml
@@ -1,15 +1,4 @@
 ---
-- name: check that hawkular_metrics_hostname is set
-  fail: msg='the openshift_metrics_hawkular_metrics_hostname variable is required'
-  when: openshift_metrics_hawkular_metrics_hostname is not defined
-
-- name: check the value of openshift_metrics_cassandra_storage_type
-  fail:
-    msg: >
-      openshift_metrics_cassandra_storage_type ({{ openshift_metrics_cassandra_storage_type }})
-      is invalid, must be one of: emptydir, pv, dynamic
-  when: openshift_metrics_cassandra_storage_type not in openshift_metrics_cassandra_storage_types
-
 - name: Create temp directory for doing work in
   command: mktemp -td openshift-metrics-ansible-XXXXXX
   register: mktemp
@@ -33,11 +22,3 @@
 
 - include: "{{role_path}}/tasks/uninstall_metrics.yaml"
   when: not openshift_metrics_install_metrics | default(false) | bool
-
-- name: create objects
-  command: >
-    {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }}
-    --config={{ mktemp.stdout }}/admin.kubeconfig
-    apply -f {{ item }}
-  with_fileglob:
-  - "{{ mktemp.stdout }}/templates/*.yaml"
diff --git a/roles/openshift_metrics/tasks/scale.yaml b/roles/openshift_metrics/tasks/scale.yaml
new file mode 100644
index 000000000..031336a01
--- /dev/null
+++ b/roles/openshift_metrics/tasks/scale.yaml
@@ -0,0 +1,27 @@
+---
+- shell: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig get {{object}}
+    --template='{{ '{{.spec.replicas}}' }}' -n {{openshift_metrics_project}}
+  register: replica_count
+  failed_when: "replica_count.rc == 1 and 'exists' not in replica_count.stderr"
+  when: not ansible_check_mode
+
+- shell: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig scale {{object}}
+    --replicas={{desired}} -n {{openshift_metrics_project}}
+  register: scale_result
+  failed_when: scale_result.rc == 1 and 'exists' not in scale_result.stderr
+  when:
+  - replica_count.stdout != desired
+  - not ansible_check_mode
+
+- name: Waiting for {{object}} to scale to {{desired}}
+  shell: >
+    {{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig describe {{object}} -n {{openshift_metrics_project}} | awk -v statusrx='Pods Status:' '$0 ~ statusrx {print $3}'
+  register: replica_counts
+  until: replica_counts.stdout.find("{{desired}}") != -1
+  retries: 30
+  delay: 10
+  when:
+    - replica_count.stdout != desired
+    - not ansible_check_mode
diff --git a/roles/openshift_metrics/tasks/start_metrics.yaml b/roles/openshift_metrics/tasks/start_metrics.yaml
new file mode 100644
index 000000000..99d593dd7
--- /dev/null
+++ b/roles/openshift_metrics/tasks/start_metrics.yaml
@@ -0,0 +1,52 @@
+---
+- shell: >
+    {{openshift.common.client_binary}} 
+    --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc 
+    -l metrics-infra=hawkular-cassandra 
+    -o name 
+    -n {{openshift_metrics_project}}
+  register: metrics_cassandra_rc
+
+- name: Start Hawkular Cassandra
+  include: scale.yaml
+  vars:
+    desired: 1
+  with_items: "{{metrics_cassandra_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
+
+- shell: >
+    {{openshift.common.client_binary}} 
+    --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc 
+    -l metrics-infra=hawkular-metrics 
+    -o name 
+    -n {{openshift_metrics_project}}
+  register: metrics_metrics_rc
+
+- name: Start Hawkular Metrics
+  include: scale.yaml
+  vars:
+    desired: "{{openshift_metrics_hawkular_replicas}}"
+  with_items: "{{metrics_metrics_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
+
+- shell: >
+    {{openshift.common.client_binary}} 
+    --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc 
+    -l metrics-infra=heapster
+    -o name 
+    -n {{openshift_metrics_project}}
+  register: metrics_heapster_rc
+  check_mode: no
+
+- name: Start Heapster
+  include: scale.yaml
+  vars:
+    desired: 1
+  with_items: "{{metrics_heapster_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
diff --git a/roles/openshift_metrics/tasks/stop_metrics.yaml b/roles/openshift_metrics/tasks/stop_metrics.yaml
new file mode 100644
index 000000000..79556e923
--- /dev/null
+++ b/roles/openshift_metrics/tasks/stop_metrics.yaml
@@ -0,0 +1,56 @@
+---
+- shell: >
+    {{openshift.common.client_binary}} 
+    --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc 
+    -l metrics-infra=heapster
+    -o name 
+    -n {{openshift_metrics_project}}
+  register: metrics_heapster_rc
+  changed_when: "'No resources found' not in metrics_heapster_rc.stderr"
+  check_mode: no
+
+- name: Stop Heapster
+  include: scale.yaml
+  vars:
+    desired: 0
+  with_items: "{{metrics_heapster_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
+
+- shell: >
+    {{openshift.common.client_binary}} 
+    --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc
+    -l metrics-infra=hawkular-metrics
+    -o name 
+    -n {{openshift_metrics_project}}
+  register: metrics_hawkular_rc
+  changed_when: "'No resources found' not in metrics_hawkular_rc.stderr"
+
+- name: Stop Hawkular Metrics
+  include: scale.yaml
+  vars:
+    desired: 0
+  with_items: "{{metrics_hawkular_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
+
+- shell: >
+    {{openshift.common.client_binary}} --config={{mktemp.stdout}}/admin.kubeconfig 
+    get rc
+    -o name
+    -l metrics-infra=hawkular-cassandra 
+    -n {{openshift_metrics_project}}
+  register: metrics_cassandra_rc
+  changed_when: "'No resources found' not in metrics_cassandra_rc.stderr"
+
+- name: Stop Hawkular Cassandra
+  include: scale.yaml
+  vars:
+    desired: 0
+  with_items: "{{metrics_cassandra_rc.stdout_lines}}"
+  loop_control:
+    loop_var: object
+  when: metrics_cassandra_rc is defined
+
diff --git a/roles/openshift_metrics/tasks/uninstall_metrics.yaml b/roles/openshift_metrics/tasks/uninstall_metrics.yaml
index cf9b5171c..8a6be6237 100644
--- a/roles/openshift_metrics/tasks/uninstall_metrics.yaml
+++ b/roles/openshift_metrics/tasks/uninstall_metrics.yaml
@@ -1,14 +1,19 @@
 ---
+- name: stop metrics
+  include: stop_metrics.yaml
+
 - name: remove metrics components
   command: >
     {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }} --config={{ mktemp.stdout }}/admin.kubeconfig
-    delete --selector=metrics-infra
+    delete --ignore-not-found --selector=metrics-infra
     all,sa,secrets,templates,routes,pvc,rolebindings,clusterrolebindings
   register: delete_metrics
   changed_when: "delete_metrics.stdout != 'No resources found'"
+
 - name: remove rolebindings
   command: >
     {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }} --config={{ mktemp.stdout }}/admin.kubeconfig
     delete --ignore-not-found
     rolebinding/hawkular-view
     clusterrolebinding/heapster-cluster-reader
+  changed_when: "delete_metrics.stdout != 'No resources found'"
diff --git a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
index 7ce1a6a87..9a1c446cd 100644
--- a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
@@ -9,7 +9,7 @@ metadata:
 spec:
   selector:
     name: hawkular-cassandra-{{ node }}
-  replicas: 1
+  replicas: 0
   template:
     version: v1
     metadata:
diff --git a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
index 4314800a3..1397276e6 100644
--- a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
@@ -8,7 +8,7 @@ metadata:
 spec:
   selector:
     name: hawkular-metrics
-  replicas: 1
+  replicas: 0
   template:
     version: v1
     metadata:
diff --git a/roles/openshift_metrics/templates/heapster.j2 b/roles/openshift_metrics/templates/heapster.j2
index 04fb76982..f64c6696e 100644
--- a/roles/openshift_metrics/templates/heapster.j2
+++ b/roles/openshift_metrics/templates/heapster.j2
@@ -8,7 +8,7 @@ metadata:
 spec:
   selector:
     name: heapster
-  replicas: 1
+  replicas: 0
   template:
     version: v1
     metadata: