Custom certificates (#5)

* Generate secrets on a persistent directory.

* Split certificate generation files.

* Custom certificates.

* Minor fixes.

- use `slurp` instead of `shell: base64`
- fix route hostname

* Updates on origin-metrics.

11 files changed, 330 insertions, 258 deletions

diff --git a/roles/openshift_metrics/README.md b/roles/openshift_metrics/README.md
index b79b472d3..092844870 100644
--- a/roles/openshift_metrics/README.md
+++ b/roles/openshift_metrics/README.md
@@ -55,6 +55,9 @@ For default values, see [`defaults/main.yaml`](defaults/main.yaml).
   override this, make sure to add `system:master-proxy` to the list in order to
   allow horizontal pod autoscaling to function properly.
 
+- `openshift_metrics_startup_timeout`: How long in seconds we should wait until
+  Hawkular Metrics and Heapster starts up before attempting a restart.
+
 - `openshift_metrics_duration`: How many days metrics should be stored for.
 
 - `openshift_metrics_resolution`: How often metrics should be gathered.
diff --git a/roles/openshift_metrics/defaults/main.yaml b/roles/openshift_metrics/defaults/main.yaml
index 8d2ff8a62..4b5ecadbf 100644
--- a/roles/openshift_metrics/defaults/main.yaml
+++ b/roles/openshift_metrics/defaults/main.yaml
@@ -3,12 +3,15 @@ openshift_metrics_image_prefix: docker.io/openshift/origin-
 openshift_metrics_image_version: latest
 openshift_metrics_master_url: https://kubernetes.default.svc.cluster.local
 openshift_metrics_project: openshift-infra
+openshift_metrics_startup_timeout: 500
 
 openshift_metrics_hawkular_user_write_access: False
 openshift_metrics_hawkular_cassandra_nodes: 1
 openshift_metrics_hawkular_cassandra_storage_type: emptydir
 openshift_metrics_hawkular_cassandra_pv_prefix: metrics-cassandra
 openshift_metrics_hawkular_cassandra_pv_size: 10Gi
+openshift_metrics_certs_dir: >
+  {{ openshift.common.config_base }}/master/metrics
 
 openshift_metrics_heapster_standalone: False
 openshift_metrics_heapster_allowed_users: system:master-proxy
diff --git a/roles/openshift_metrics/tasks/generate_certificates.yaml b/roles/openshift_metrics/tasks/generate_certificates.yaml
index 9f6a3348e..92ce919a1 100644
--- a/roles/openshift_metrics/tasks/generate_certificates.yaml
+++ b/roles/openshift_metrics/tasks/generate_certificates.yaml
@@ -1,233 +1,22 @@
 ---
-# TODO idempotency?
-# TODO support providing custom certificates
 - name: create certificate output directory
   file:
-    path: "{{ mktemp.stdout }}/certs"
+    path: "{{ openshift_metrics_certs_dir }}"
     state: directory
     mode: 0700
+- name: list existing secrets
+  command: >
+    {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }}
+    get secrets -o name
+  register: metrics_secrets
+  changed_when: false
 - name: generate ca certificate chain
   shell: >
     {{ openshift.common.admin_binary }} ca create-signer-cert
-    --key='{{ mktemp.stdout }}/certs/ca.key'
-    --cert='{{ mktemp.stdout }}/certs/ca.crt'
-    --serial='{{ mktemp.stdout }}/certs/ca.serial.txt'
+    --key='{{ openshift_metrics_certs_dir }}/ca.key'
+    --cert='{{ openshift_metrics_certs_dir }}/ca.crt'
+    --serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
     --name="metrics-signer@$(date +%s)"
-- name: generate heapster key/cert
-  command: >
-    {{ openshift.common.admin_binary }} ca create-server-cert
-    --key='{{ mktemp.stdout }}/certs/heapster.key'
-    --cert='{{ mktemp.stdout }}/certs/heapster.cert'
-    --hostnames=heapster
-    --signer-cert='{{ mktemp.stdout }}/certs/ca.crt'
-    --signer-key='{{ mktemp.stdout }}/certs/ca.key'
-    --signer-serial='{{ mktemp.stdout }}/certs/ca.serial.txt'
-# TODO maybe there's an easier way to get the service accounts' ca crt?
-- name: get heapster service account secrets
-  shell: >
-    {{ openshift.common.client_binary }} -n '{{ openshift_metrics_project }}'
-    get serviceaccount/default
-    --template '{{ '{{range .secrets}}{{println .name}}{{end}}' }}'
-    | grep ^default-token-
-  register: sa_secret
-- name: get heapster service account ca
-  command: >
-    {{ openshift.common.client_binary }} -n '{{ openshift_metrics_project }}'
-    get 'secret/{{ sa_secret.stdout }}'
-    --template '{{ '{{index .data "ca.crt"}}' }}'
-  register: sa_secret
-- name: read files for the heapster secret
-  command: base64 --wrap 0 "{{ mktemp.stdout }}/certs/heapster.{{ item }}"
-  register: heapster_secret
-  with_items:
-  - cert
-  - key
-- name: generate heapster secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/heapster_secrets.yaml"
-  vars:
-    name: heapster-secrets
-    labels:
-      metrics-infra: heapster
-    data:
-      heapster.cert: "{{ heapster_secret.results[0].stdout }}"
-      heapster.key: "{{ heapster_secret.results[1].stdout }}"
-      heapster.client-ca: "{{ sa_secret.stdout }}"
-      heapster.allowed-users: "{{ openshift_metrics_heapster_allowed_users|b64encode }}"
-- name: generate hawkular-metrics certificates
-  include: setup_certificate.yaml
-  vars:
-    component: hawkular-metrics
-    hostnames: "hawkular-metrics,{{ openshift_metrics_hawkular_metrics_hostname }}"
-- name: generate hawkular-cassandra certificates
-  include: setup_certificate.yaml
-  vars:
-    component: hawkular-cassandra
-    hostnames: hawkular-cassandra
-# TODO keytool as dependency?  move key/trust store generation to containers?
-- name: import the hawkular metrics cert into the cassandra truststore
-  shell: >
-    keytool -noprompt -import -v -trustcacerts
-    -alias hawkular-metrics
-    -file '{{ mktemp.stdout|quote }}/certs/hawkular-metrics.cert'
-    -keystore '{{ mktemp.stdout|quote }}/certs/hawkular-cassandra.truststore'
-    -storepass
-    "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")"
-- name: import the hawkular cassandra cert into the hawkular metrics truststore
-  shell: >
-    keytool -noprompt -import -v -trustcacerts
-    -alias hawkular-cassandra
-    -file '{{ mktemp.stdout }}/certs/hawkular-cassandra.cert'
-    -keystore '{{ mktemp.stdout }}/certs/hawkular-metrics.truststore'
-    -storepass
-    "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-metrics-truststore.pwd")"
-- name: import the hawkular cassandra cert into the cassandra truststore
-  shell: >
-    keytool -noprompt -import -v -trustcacerts
-    -alias hawkular-cassandra
-    -file '{{ mktemp.stdout }}/certs/hawkular-cassandra.cert'
-    -keystore '{{ mktemp.stdout }}/certs/hawkular-cassandra.truststore'
-    -storepass
-    "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")"
-- name: import the ca certificate into the cassandra truststore
-  shell: >
-    keytool -noprompt -import -v -trustcacerts
-    -alias '{{ item }}'
-    -file '{{ mktemp.stdout }}/certs/ca.crt'
-    -keystore '{{ mktemp.stdout }}/certs/hawkular-cassandra.truststore'
-    -storepass
-    "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-cassandra-truststore.pwd")"
-  with_items:
-  - ca
-  - metricca
-  - cassandraca
-- name: import the ca certificate into the hawkular metrics truststore
-  shell: >
-    keytool -noprompt -import -v -trustcacerts
-    -alias '{{ item }}'
-    -file '{{ mktemp.stdout }}/certs/ca.crt'
-    -keystore '{{ mktemp.stdout }}/certs/hawkular-metrics.truststore'
-    -storepass
-    "$(< "{{ mktemp.stdout|quote }}/certs/hawkular-metrics-truststore.pwd")"
-  with_items:
-  - ca
-  - metricca
-  - cassandraca
-- name: generate password for htpasswd file for hawkular metrics
-  shell: cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c15
-  register: hawkular_metrics_password
-- name: generate password for hawkular metrics jgroups
-  shell: cat /dev/urandom | tr -dc _A-Z-a-z-0-9 | head -c15
-  register: hawkular_metrics_jgroups_password
-- name: generate htpasswd file for hawkular metrics
-  shell: >
-    htpasswd -cb
-    "{{ mktemp.stdout|quote }}/certs/hawkular-metrics.htpasswd" hawkular
-    '{{ hawkular_metrics_password.stdout }}'
-- name: generate the jgroups keystore
-  command: >
-    keytool -genseckey -alias hawkular
-    -keypass {{ hawkular_metrics_jgroups_password.stdout }}
-    -storepass {{ hawkular_metrics_jgroups_password.stdout }}
-    -keyalg Blowfish -keysize 56 -storetype JCEKS
-    -keystore {{ mktemp.stdout }}/certs/hawkular-jgroups.keystore
-- name: read files for the hawkular-metrics secret
-  command: >
-    base64 --wrap 0 "{{ mktemp.stdout }}/certs/{{ item }}"
-  register: hawkular_metrics_secret
-  with_items:
-  - hawkular-metrics.keystore
-  - hawkular-metrics-keystore.pwd
-  - hawkular-metrics.truststore
-  - hawkular-metrics-truststore.pwd
-  - hawkular-metrics.htpasswd
-  - hawkular-metrics.cert
-  - ca.crt
-  - hawkular-cassandra.keystore
-  - hawkular-cassandra-keystore.pwd
-  - hawkular-cassandra.truststore
-  - hawkular-cassandra-truststore.pwd
-  - hawkular-cassandra.pem
-  - hawkular-cassandra.cert
-  - hawkular-jgroups.keystore
-- name: generate hawkular-metrics-secrets secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_secrets.yaml"
-  vars:
-    name: hawkular-metrics-secrets
-    labels:
-      metrics-infra: hawkular-metrics
-    data:
-      hawkular-metrics.keystore: >
-        "{{ hawkular_metrics_secret.results[0].stdout }}"
-      hawkular-metrics.keystore.password: >
-        "{{ hawkular_metrics_secret.results[1].stdout }}"
-      hawkular-metrics.truststore: >
-        "{{ hawkular_metrics_secret.results[2].stdout }}"
-      hawkular-metrics.truststore.password: >
-        "{{ hawkular_metrics_secret.results[3].stdout }}"
-      hawkular-metrics.keystore.alias: "{{ 'hawkular-metrics'|b64encode }}"
-      hawkular-metrics.htpasswd.file: >
-        "{{ hawkular_metrics_secret.results[4].stdout }}"
-      hawkular-metrics.jgroups.keystore.password: >
-        "{{ hawkular_metrics_jgroups_password.stdout|b64encode }}"
-      hawkular-metrics.jgroups.keystore: >
-        "{{ hawkular_metrics_secret.results[13].stdout }}"
-      hawkular-metrics.jgroups.alias: "{{ 'hawkular'|b64encode }}"
-- name: generate hawkular-metrics-certificate secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_certificate.yaml"
-  vars:
-    name: hawkular-metrics-certificate
-    labels:
-      metrics-infra: hawkular-metrics
-    data:
-      hawkular-metrics.certificate: >
-        "{{ hawkular_metrics_secret.results[5].stdout }}"
-      hawkular-metrics-ca.certificate: >
-        "{{ hawkular_metrics_secret.results[6].stdout }}"
-- name: generate hawkular-metrics-account secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_account.yaml"
-  vars:
-    name: hawkular-metrics-account
-    labels:
-      metrics-infra: hawkular-metrics
-    data:
-      hawkular-metrics.username: "{{ 'hawkular'|b64encode }}"
-      hawkular-metrics.password: >
-        "{{ hawkular_metrics_password.stdout|b64encode }}"
-- name: generate cassandra secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/cassandra_secrets.yaml"
-  vars:
-    name: hawkular-cassandra-secrets
-    labels:
-      metrics-infra: hawkular-cassandra
-    data:
-      cassandra.keystore: "{{ hawkular_metrics_secret.results[7].stdout }}"
-      cassandra.keystore.password: >
-        {{ hawkular_metrics_secret.results[8].stdout }}
-      cassandra.keystore.alias: "{{ 'hawkular-cassandra'|b64encode }}"
-      cassandra.truststore: "{{ hawkular_metrics_secret.results[9].stdout }}"
-      cassandra.truststore.password: >
-        {{ hawkular_metrics_secret.results[10].stdout }}
-      cassandra.pem: "{{ hawkular_metrics_secret.results[10].stdout }}"
-- name: generate cassandra-certificate secret template
-  template:
-    src: secret.j2
-    dest: "{{ mktemp.stdout }}/templates/cassandra_certificate.yaml"
-  vars:
-    name: hawkular-cassandra-certificate
-    labels:
-      metrics-infra: hawkular-cassandra
-    data:
-      cassandra.certificate: >
-        {{ hawkular_metrics_secret.results[11].stdout }}
-      cassandra-ca.certificate: >
-        {{ hawkular_metrics_secret.results[7].stdout }}
+  when: not '{{ openshift_metrics_certs_dir }}/ca.key'|exists
+- include: generate_heapster_certificates.yaml
+- include: generate_hawkular_certificates.yaml
diff --git a/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml b/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml
new file mode 100644
index 000000000..4e032ca7e
--- /dev/null
+++ b/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml
@@ -0,0 +1,227 @@
+---
+- name: generate hawkular-metrics certificates
+  include: setup_certificate.yaml
+  vars:
+    component: hawkular-metrics
+    hostnames: "hawkular-metrics,{{ openshift_metrics_hawkular_metrics_hostname }}"
+- name: generate hawkular-cassandra certificates
+  include: setup_certificate.yaml
+  vars:
+    component: hawkular-cassandra
+    hostnames: hawkular-cassandra
+- name: check existing aliases on the hawkular-cassandra truststore
+  shell: >
+    keytool -noprompt -list
+    -keystore {{ openshift_metrics_certs_dir }}/hawkular-cassandra.truststore
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-cassandra-truststore.pwd')"
+    | sed -n '7~2s/,.*$//p'
+  register: hawkular_cassandra_truststore_aliases
+  changed_when: false
+- name: check existing aliases on the hawkular-metrics truststore
+  shell: >
+    keytool -noprompt -list
+    -keystore {{ openshift_metrics_certs_dir }}/hawkular-metrics.truststore
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-metrics-truststore.pwd')"
+    | sed -n '7~2s/,.*$//p'
+  register: hawkular_metrics_truststore_aliases
+  changed_when: false
+- name: import the hawkular metrics cert into the cassandra truststore
+  shell: >
+    keytool -noprompt -import -v -trustcacerts
+    -alias hawkular-metrics
+    -file '{{ openshift_metrics_certs_dir }}/hawkular-metrics.crt'
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-cassandra.truststore'
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-cassandra-truststore.pwd')"
+  when: >
+    'hawkular-metrics' not in
+    hawkular_cassandra_truststore_aliases.stdout_lines
+- name: import the hawkular cassandra cert into the hawkular metrics truststore
+  shell: >
+    keytool -noprompt -import -v -trustcacerts
+    -alias hawkular-cassandra
+    -file '{{ openshift_metrics_certs_dir }}/hawkular-cassandra.crt'
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-metrics.truststore'
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-metrics-truststore.pwd')"
+  when: >
+    'hawkular-cassandra' not in
+    hawkular_metrics_truststore_aliases.stdout_lines
+- name: import the hawkular cassandra cert into the cassandra truststore
+  shell: >
+    keytool -noprompt -import -v -trustcacerts
+    -alias hawkular-cassandra
+    -file '{{ openshift_metrics_certs_dir }}/hawkular-cassandra.crt'
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-cassandra.truststore'
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-cassandra-truststore.pwd')"
+  when: >
+    'hawkular-cassandra' not in
+    hawkular_cassandra_truststore_aliases.stdout_lines
+- name: import the ca certificate into the cassandra truststore
+  shell: >
+    keytool -noprompt -import -v -trustcacerts
+    -alias '{{ item }}'
+    -file '{{ openshift_metrics_certs_dir }}/ca.crt'
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-cassandra.truststore'
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-cassandra-truststore.pwd')"
+  with_items:
+  - ca
+  - metricca
+  - cassandraca
+  when: item not in hawkular_cassandra_truststore_aliases.stdout_lines
+- name: import the ca certificate into the hawkular metrics truststore
+  shell: >
+    keytool -noprompt -import -v -trustcacerts
+    -alias '{{ item }}'
+    -file '{{ openshift_metrics_certs_dir }}/ca.crt'
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-metrics.truststore'
+    -storepass "$(<
+    '{{ openshift_metrics_certs_dir }}/hawkular-metrics-truststore.pwd')"
+  with_items:
+  - ca
+  - metricca
+  - cassandraca
+  when: item not in hawkular_metrics_truststore_aliases.stdout_lines
+- name: generate password for hawkular metrics and jgroups
+  shell: >
+    tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15
+    > '{{ openshift_metrics_certs_dir }}/{{ item }}.pwd'
+  with_items:
+  - hawkular-metrics
+  - hawkular-jgroups-keystore
+  when: not '{{ openshift_metrics_certs_dir }}/{{ item }}.pwd'|exists
+- name: generate htpasswd file for hawkular metrics
+  shell: >
+    htpasswd -ci
+    '{{ openshift_metrics_certs_dir }}/hawkular-metrics.htpasswd' hawkular
+    < '{{ openshift_metrics_certs_dir }}/hawkular-metrics.pwd'
+  when: >
+    not '{{ openshift_metrics_certs_dir }}/hawkular-metrics.htpasswd'|exists
+- name: generate the jgroups keystore
+  shell: >
+    p=$(< '{{ openshift_metrics_certs_dir }}/hawkular-jgroups-keystore.pwd' )
+    &&
+    keytool -genseckey -alias hawkular
+    -keypass "$p" -storepass "$p" -keyalg Blowfish -keysize 56 -storetype JCEKS
+    -keystore '{{ openshift_metrics_certs_dir }}/hawkular-jgroups.keystore'
+  when: >
+    not '{{ openshift_metrics_certs_dir }}/hawkular-jgroups.keystore'|exists
+- name: read files for the hawkular-metrics secret
+  shell: >
+    printf '%s: ' '{{ item }}'
+    && base64 --wrap 0 '{{ openshift_metrics_certs_dir }}/{{ item }}'
+  register: hawkular_secrets
+  with_items:
+  - ca.crt
+  - hawkular-metrics.crt
+  - hawkular-metrics.keystore
+  - hawkular-metrics-keystore.pwd
+  - hawkular-metrics.truststore
+  - hawkular-metrics-truststore.pwd
+  - hawkular-metrics.pwd
+  - hawkular-metrics.htpasswd
+  - hawkular-jgroups.keystore
+  - hawkular-jgroups-keystore.pwd
+  - hawkular-cassandra.crt
+  - hawkular-cassandra.pem
+  - hawkular-cassandra.keystore
+  - hawkular-cassandra-keystore.pwd
+  - hawkular-cassandra.truststore
+  - hawkular-cassandra-truststore.pwd
+  changed_when: false
+- set_fact:
+    hawkular_secrets: |
+      {{ hawkular_secrets.results|map(attribute='stdout')|join('
+      ')|from_yaml }}
+- name: generate hawkular-metrics-secrets secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_secrets.yaml"
+  vars:
+    name: hawkular-metrics-secrets
+    labels:
+      metrics-infra: hawkular-metrics
+    data:
+      hawkular-metrics.keystore: >
+        {{ hawkular_secrets['hawkular-metrics.keystore'] }}
+      hawkular-metrics.keystore.password: >
+        {{ hawkular_secrets['hawkular-metrics-keystore.pwd'] }}
+      hawkular-metrics.truststore: >
+        {{ hawkular_secrets['hawkular-metrics.truststore'] }}
+      hawkular-metrics.truststore.password: >
+        {{ hawkular_secrets['hawkular-metrics-truststore.pwd'] }}
+      hawkular-metrics.keystore.alias: "{{ 'hawkular-metrics'|b64encode }}"
+      hawkular-metrics.htpasswd.file: >
+        {{ hawkular_secrets['hawkular-metrics.htpasswd'] }}
+      hawkular-metrics.jgroups.keystore: >
+        {{ hawkular_secrets['hawkular-jgroups.keystore'] }}
+      hawkular-metrics.jgroups.keystore.password: >
+        {{ hawkular_secrets['hawkular-jgroups-keystore.pwd'] }}
+      hawkular-metrics.jgroups.alias: "{{ 'hawkular'|b64encode }}"
+  when: name not in metrics_secrets.stdout_lines
+- name: generate hawkular-metrics-certificate secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_certificate.yaml"
+  vars:
+    name: hawkular-metrics-certificate
+    labels:
+      metrics-infra: hawkular-metrics
+    data:
+      hawkular-metrics.certificate: >
+        {{ hawkular_secrets['hawkular-metrics.crt'] }}
+      hawkular-metrics-ca.certificate: >
+        {{ hawkular_secrets['ca.crt'] }}
+  when: name not in metrics_secrets.stdout_lines
+- name: generate hawkular-metrics-account secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/hawkular_metrics_account.yaml"
+  vars:
+    name: hawkular-metrics-account
+    labels:
+      metrics-infra: hawkular-metrics
+    data:
+      hawkular-metrics.username: "{{ 'hawkular'|b64encode }}"
+      hawkular-metrics.password: >
+        {{ hawkular_secrets['hawkular-metrics.pwd'] }}
+  when: name not in metrics_secrets.stdout_lines
+- name: generate cassandra secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/cassandra_secrets.yaml"
+  vars:
+    name: hawkular-cassandra-secrets
+    labels:
+      metrics-infra: hawkular-cassandra
+    data:
+      cassandra.keystore: >
+        {{ hawkular_secrets['hawkular-cassandra.keystore'] }}
+      cassandra.keystore.password: >
+        {{ hawkular_secrets['hawkular-cassandra-keystore.pwd'] }}
+      cassandra.keystore.alias: "{{ 'hawkular-cassandra'|b64encode }}"
+      cassandra.truststore: >
+        {{ hawkular_secrets['hawkular-cassandra.truststore'] }}
+      cassandra.truststore.password: >
+        {{ hawkular_secrets['hawkular-cassandra-truststore.pwd'] }}
+      cassandra.pem: >
+        {{ hawkular_secrets['hawkular-cassandra.pem'] }}
+  when: name not in metrics_secrets
+- name: generate cassandra-certificate secret template
+  template:
+    src: secret.j2
+    dest: "{{ mktemp.stdout }}/templates/cassandra_certificate.yaml"
+  vars:
+    name: hawkular-cassandra-certificate
+    labels:
+      metrics-infra: hawkular-cassandra
+    data:
+      cassandra.certificate: >
+        {{ hawkular_secrets['hawkular-cassandra.crt'] }}
+      cassandra-ca.certificate: >
+        {{ hawkular_secrets['hawkular-cassandra.pem'] }}
+  when: name not in metrics_secrets.stdout_lines
diff --git a/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml b/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml
new file mode 100644
index 000000000..2fc449520
--- /dev/null
+++ b/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml
@@ -0,0 +1,39 @@
+---
+- name: generate heapster key/cert
+  command: >
+    {{ openshift.common.admin_binary }} ca create-server-cert
+    --key='{{ openshift_metrics_certs_dir }}/heapster.key'
+    --cert='{{ openshift_metrics_certs_dir }}/heapster.cert'
+    --hostnames=heapster
+    --signer-cert='{{ openshift_metrics_certs_dir }}/ca.crt'
+    --signer-key='{{ openshift_metrics_certs_dir }}/ca.key'
+    --signer-serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
+  when: not '{{ openshift_metrics_certs_dir }}/heapster.key'|exists
+- when: "'secret/heapster-secrets' not in metrics_secrets.stdout_lines"
+  block:
+  - name: read files for the heapster secret
+    slurp: src={{ item }}
+    register: heapster_secret
+    with_items:
+    - "{{ openshift_metrics_certs_dir }}/heapster.cert"
+    - "{{ openshift_metrics_certs_dir }}/heapster.key"
+    - "{{ client_ca }}"
+    vars:
+      custom_ca: "{{ openshift_metrics_certs_dir }}/heapster_client_ca.crt"
+      default_ca: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
+      client_ca: "{{ custom_ca|exists|ternary(custom_ca, default_ca) }}"
+  - name: generate heapster secret template
+    template:
+      src: secret.j2
+      dest: "{{ mktemp.stdout }}/templates/heapster_secrets.yaml"
+      force: no
+    vars:
+      name: heapster-secrets
+      labels:
+        metrics-infra: heapster
+      data:
+        heapster.cert: "{{ heapster_secret.results[0].content }}"
+        heapster.key: "{{ heapster_secret.results[1].content }}"
+        heapster.client-ca: "{{ heapster_secret.results[2].content }}"
+        heapster.allowed-users: >
+          {{ openshift_metrics_heapster_allowed_users|b64encode }}
diff --git a/roles/openshift_metrics/tasks/install_hawkular.yaml b/roles/openshift_metrics/tasks/install_hawkular.yaml
index 9a39cce34..d7a029fa8 100644
--- a/roles/openshift_metrics/tasks/install_hawkular.yaml
+++ b/roles/openshift_metrics/tasks/install_hawkular.yaml
@@ -39,6 +39,9 @@
     size: "{{ openshift_metrics_hawkular_cassandra_pv_size }}"
   with_sequence: count={{ openshift_metrics_hawkular_cassandra_nodes }}
   when: openshift_metrics_hawkular_cassandra_storage_type == 'dynamic'
+- name: read hawkular-metrics route destination ca certificate
+  slurp: src={{ openshift_metrics_certs_dir }}/ca.crt
+  register: metrics_route_dest_ca_cert
 - name: generate the hawkular-metrics route
   template:
     src: route.j2
@@ -47,11 +50,10 @@
     name: hawkular-metrics
     labels:
       metrics-infra: hawkular-metrics
-    host: hawkular-metrics.example.com
+    host: "{{ openshift_metrics_hawkular_metrics_hostname }}"
     to:
       kind: Service
       name: hawkular-metrics
     tls:
       termination: reencrypt
-      destination_ca_certificate: >
-        {{ hawkular_metrics_secret.results[6].stdout|b64decode }}
+      destination_ca_certificate: "{{ metrics_route_dest_ca_cert.content }}"
diff --git a/roles/openshift_metrics/tasks/install_metrics.yaml b/roles/openshift_metrics/tasks/install_metrics.yaml
index 34b4a47fe..5d95fa112 100644
--- a/roles/openshift_metrics/tasks/install_metrics.yaml
+++ b/roles/openshift_metrics/tasks/install_metrics.yaml
@@ -11,7 +11,7 @@
   file: path={{mktemp.stdout}}/templates state=directory mode=0755
   changed_when: False
 
+- include: generate_certificates.yaml
 - include: generate_serviceaccounts.yaml
 - include: generate_services.yaml
-- include: generate_certificates.yaml
 - include: generate_rolebindings.yaml
diff --git a/roles/openshift_metrics/tasks/setup_certificate.yaml b/roles/openshift_metrics/tasks/setup_certificate.yaml
index 46ac4ea7f..d6ee4167b 100644
--- a/roles/openshift_metrics/tasks/setup_certificate.yaml
+++ b/roles/openshift_metrics/tasks/setup_certificate.yaml
@@ -2,49 +2,51 @@
 - name: generate {{ component }} keys
   command: >
     {{ openshift.common.admin_binary }} ca create-server-cert
-    --key='{{ mktemp.stdout }}/certs/{{ component }}.key'
-    --cert='{{ mktemp.stdout }}/certs/{{ component }}.crt'
+    --key='{{ openshift_metrics_certs_dir }}/{{ component }}.key'
+    --cert='{{ openshift_metrics_certs_dir }}/{{ component }}.crt'
     --hostnames='{{ hostnames }}'
-    --signer-cert='{{ mktemp.stdout }}/certs/ca.crt'
-    --signer-key='{{ mktemp.stdout }}/certs/ca.key'
-    --signer-serial='{{ mktemp.stdout }}/certs/ca.serial.txt'
+    --signer-cert='{{ openshift_metrics_certs_dir }}/ca.crt'
+    --signer-key='{{ openshift_metrics_certs_dir }}/ca.key'
+    --signer-serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
+  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.key'|exists
 - name: generate {{ component }} certificate
   shell: >
     cat
-    '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.key'
-    '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.crt'
-    > '{{ mktemp.stdout|quote }}/certs/{{ component|quote }}.pem'
+    '{{ openshift_metrics_certs_dir }}/{{ component|quote }}.key'
+    '{{ openshift_metrics_certs_dir }}/{{ component|quote }}.crt'
+    > '{{ openshift_metrics_certs_dir }}/{{ component|quote }}.pem'
+  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'|exists
 - name: generate random password for the {{ component }} keystore
-  shell: tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15
-  register: keystore_pwd
-- name: create the password file for {{ component }}
   shell: >
-    echo '{{ keystore_pwd.stdout|quote }}'
-    > '{{ mktemp.stdout }}/certs/{{ component|quote }}-keystore.pwd'
+    tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15
+    > '{{ openshift_metrics_certs_dir }}/{{ component|quote }}-keystore.pwd'
+  when: >
+    not '{{ openshift_metrics_certs_dir }}/{{ component }}-keystore.pwd'|exists
 - name: create the {{ component }} pkcs12 from the pem file
   command: >
     openssl pkcs12 -export
-    -in '{{ mktemp.stdout }}/certs/{{ component }}.pem'
-    -out '{{ mktemp.stdout }}/certs/{{ component }}.pkcs12'
+    -in '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'
+    -out '{{ openshift_metrics_certs_dir }}/{{ component }}.pkcs12'
     -name '{{ component }}' -noiter -nomaciter
-    -password 'pass:{{ keystore_pwd.stdout }}'
+    -password
+    'file:{{ openshift_metrics_certs_dir }}/{{ component }}-keystore.pwd'
+  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.pkcs12'|exists
 - name: create the {{ component }} keystore from the pkcs12 file
-  command: >
+  shell: >
+    p=$(< {{ openshift_metrics_certs_dir }}/{{ component }}-keystore.pwd)
+    &&
     keytool -v -importkeystore
-    -srckeystore '{{ mktemp.stdout }}/certs/{{ component }}.pkcs12'
+    -srckeystore '{{ openshift_metrics_certs_dir }}/{{ component }}.pkcs12'
     -srcstoretype PKCS12
-    -destkeystore '{{ mktemp.stdout }}/certs/{{ component }}.keystore'
+    -destkeystore '{{ openshift_metrics_certs_dir }}/{{ component }}.keystore'
     -deststoretype JKS
-    -deststorepass '{{ keystore_pwd.stdout }}'
-    -srcstorepass '{{ keystore_pwd.stdout }}'
-- name: create the {{ component }} certificate
-  command: >
-    keytool -noprompt -export
-    -alias '{{ component }}'
-    -file '{{ mktemp.stdout }}/certs/{{ component }}.cert'
-    -keystore '{{ mktemp.stdout }}/certs/{{ component }}.keystore'
-    -storepass '{{ keystore_pwd.stdout }}'
+    -deststorepass "$p"
+    -srcstorepass "$p"
+  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.keystore'|exists
 - name: generate random password for the {{ component }} truststore
   shell: >
     tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c15
-    > '{{ mktemp.stdout }}/certs/{{ component|quote }}-truststore.pwd'
+    > '{{ openshift_metrics_certs_dir }}/{{ component|quote }}-truststore.pwd'
+  when: >
+    not
+    '{{ openshift_metrics_certs_dir }}/{{ component }}-truststore.pwd'|exists
diff --git a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2 b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
index 525f32859..158d0d1a3 100644
--- a/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_cassandra_rc.j2
@@ -49,6 +49,8 @@ spec:
           value: "{{ master }}"
         - name: CASSANDRA_DATA_VOLUME
           value: "/cassandra_data"
+        - name: JVM_OPTS
+          value: "-Dcassandra.commitlog.ignorereplayerrors=true"
         - name: POD_NAMESPACE
           valueFrom:
             fieldRef:
diff --git a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2 b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
index 6f1275809..647a4bfbb 100644
--- a/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
+++ b/roles/openshift_metrics/templates/hawkular_metrics_rc.j2
@@ -66,6 +66,8 @@ spec:
               fieldPath: metadata.namespace
         - name: OPENSHIFT_KUBE_PING_LABELS
           value: "metrics-infra=hawkular-metrics,name=hawkular-metrics"
+        - name: STARTUP_TIMEOUT
+          value: "{{ openshift_metrics_startup_timeout }}"
         volumeMounts:
         - name: hawkular-metrics-secrets
           mountPath: "/secrets"
diff --git a/roles/openshift_metrics/templates/heapster.j2 b/roles/openshift_metrics/templates/heapster.j2
index e4b4b9739..90227db68 100644
--- a/roles/openshift_metrics/templates/heapster.j2
+++ b/roles/openshift_metrics/templates/heapster.j2
@@ -27,7 +27,7 @@ spec:
         command:
         - "heapster-wrapper.sh"
         - "--wrapper.allowed_users_file=/secrets/heapster.allowed-users"
-        - "--source=kubernetes:{{openshift_metrics_master_url}}?useServiceAccount=true&kubeletHttps=true&kubeletPort=10250"
+        - "--source=kubernetes.summary_api:${MASTER_URL}?useServiceAccount=true&kubeletHttps=true&kubeletPort=10250"
         - "--tls_cert=/secrets/heapster.cert"
         - "--tls_key=/secrets/heapster.key"
         - "--tls_client_ca=/secrets/heapster.client-ca"
@@ -39,6 +39,9 @@ spec:
         - "--wrapper.endpoint_check=https://hawkular-metrics:443/hawkular/metrics/status"
         - "--sink=hawkular:https://hawkular-metrics:443?tenant=_system&labelToTenant=pod_namespace&labelNodeId={{openshift_metrics_node_id}}&caCert=/hawkular-cert/hawkular-metrics-ca.certificate&user=%username%&pass=%password%&filter=label(container_name:^system.slice.*|^user.slice)"
 {% endif %}
+        env:
+        - name: STARTUP_TIMEOUT
+          value: "{{ openshift_metrics_startup_timeout }}"
         volumeMounts:
         - name: heapster-secrets
           mountPath: "/secrets"