Merge pull request #3297 from jcantrill/bz_1419962_cassandra_pwd_failure

bug 1419962. fix openshift_metrics pwd issue after reinstall where ca…

9 files changed, 53 insertions, 70 deletions

diff --git a/roles/openshift_metrics/defaults/main.yaml b/roles/openshift_metrics/defaults/main.yaml
index 485cd874c..0cfbac8a9 100644
--- a/roles/openshift_metrics/defaults/main.yaml
+++ b/roles/openshift_metrics/defaults/main.yaml
@@ -42,7 +42,6 @@ openshift_metrics_resolution: 15s
 # overriding the values here
 #####
 
-openshift_metrics_certs_dir: "{{ openshift.common.config_base }}/master/metrics"
 openshift_metrics_master_url: https://kubernetes.default.svc.cluster.local
 openshift_metrics_node_id: nodename
 openshift_metrics_project: openshift-infra
diff --git a/roles/openshift_metrics/tasks/generate_certificates.yaml b/roles/openshift_metrics/tasks/generate_certificates.yaml
index 4925275e8..f7cba0093 100644
--- a/roles/openshift_metrics/tasks/generate_certificates.yaml
+++ b/roles/openshift_metrics/tasks/generate_certificates.yaml
@@ -1,11 +1,11 @@
 ---
 - name: generate ca certificate chain
-  shell: >
+  command: >
     {{ openshift.common.admin_binary }} ca create-signer-cert
     --config={{ mktemp.stdout }}/admin.kubeconfig
-    --key='{{ openshift_metrics_certs_dir }}/ca.key'
-    --cert='{{ openshift_metrics_certs_dir }}/ca.crt'
-    --serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
+    --key='{{ mktemp.stdout }}/ca.key'
+    --cert='{{ mktemp.stdout }}/ca.crt'
+    --serial='{{ mktemp.stdout }}/ca.serial.txt'
     --name="metrics-signer@$(date +%s)"
-  when: not '{{ openshift_metrics_certs_dir }}/ca.key' | exists
+
 - include: generate_hawkular_certificates.yaml
diff --git a/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml b/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml
index 9333d341c..854697abb 100644
--- a/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml
+++ b/roles/openshift_metrics/tasks/generate_hawkular_certificates.yaml
@@ -13,13 +13,13 @@
     hostnames: hawkular-cassandra
   changed_when: no
 
-- slurp: src={{ openshift_metrics_certs_dir }}/hawkular-cassandra-truststore.pwd
+- slurp: src={{ mktemp.stdout }}/hawkular-cassandra-truststore.pwd
   register: cassandra_truststore_password
 
-- slurp: src={{ openshift_metrics_certs_dir }}/hawkular-metrics-truststore.pwd
+- slurp: src={{ mktemp.stdout }}/hawkular-metrics-truststore.pwd
   register: hawkular_truststore_password
 
-- stat: path="{{openshift_metrics_certs_dir}}/{{item}}"
+- stat: path="{{mktemp.stdout}}/{{item}}"
   register: pwd_file_stat
   with_items:
   - hawkular-metrics.pwd
@@ -32,44 +32,33 @@
   with_items: "{{pwd_file_stat.results}}"
   changed_when: no
 
-- name: Create temp directory local on control node
-  local_action: command mktemp -d
-  register: local_tmp
-  changed_when: False
-
 - name: generate password for hawkular metrics and jgroups
   local_action: copy dest="{{ local_tmp.stdout}}/{{ item }}.pwd" content="{{ 15 | oo_random_word }}"
   with_items:
   - hawkular-metrics
   - hawkular-jgroups-keystore
-  when: "not pwd_files['{{ item }}.pwd'].exists"
 
 - name: generate htpasswd file for hawkular metrics
   local_action: >
     shell htpasswd -ci
     '{{ local_tmp.stdout }}/hawkular-metrics.htpasswd' hawkular
     < '{{ local_tmp.stdout }}/hawkular-metrics.pwd'
-  when: "not pwd_files['hawkular-metrics.htpasswd'].exists"
 
 - name: copy local generated passwords to target
   copy:
     src: "{{local_tmp.stdout}}/{{item}}"
-    dest: "{{openshift_metrics_certs_dir}}/{{item}}"
+    dest: "{{mktemp.stdout}}/{{item}}"
   with_items:
   - hawkular-metrics.pwd
   - hawkular-metrics.htpasswd
   - hawkular-jgroups-keystore.pwd
-  when: "not pwd_files['{{ item }}'].exists"
 
 - include: import_jks_certs.yaml
 
-- local_action: file path="{{local_tmp.stdout}}" state=absent
-  changed_when: False
-
 - name: read files for the hawkular-metrics secret
   shell: >
     printf '%s: ' '{{ item }}'
-    && base64 --wrap 0 '{{ openshift_metrics_certs_dir }}/{{ item }}'
+    && base64 --wrap 0 '{{ mktemp.stdout }}/{{ item }}'
   register: hawkular_secrets
   with_items:
   - ca.crt
diff --git a/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml b/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml
index 2449b1518..ced2df1d0 100644
--- a/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml
+++ b/roles/openshift_metrics/tasks/generate_heapster_certificates.yaml
@@ -3,13 +3,12 @@
   command: >
     {{ openshift.common.admin_binary }} ca create-server-cert
     --config={{ mktemp.stdout }}/admin.kubeconfig
-    --key='{{ openshift_metrics_certs_dir }}/heapster.key'
-    --cert='{{ openshift_metrics_certs_dir }}/heapster.cert'
+    --key='{{ mktemp.stdout }}/heapster.key'
+    --cert='{{ mktemp.stdout }}/heapster.cert'
     --hostnames=heapster
-    --signer-cert='{{ openshift_metrics_certs_dir }}/ca.crt'
-    --signer-key='{{ openshift_metrics_certs_dir }}/ca.key'
-    --signer-serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
-  when: not '{{ openshift_metrics_certs_dir }}/heapster.key' | exists
+    --signer-cert='{{ mktemp.stdout }}/ca.crt'
+    --signer-key='{{ mktemp.stdout }}/ca.key'
+    --signer-serial='{{ mktemp.stdout }}/ca.serial.txt'
 
 - when: "'secret/heapster-secrets' not in metrics_secrets.stdout_lines"
   block:
@@ -17,11 +16,11 @@
     slurp: src={{ item }}
     register: heapster_secret
     with_items:
-    - "{{ openshift_metrics_certs_dir }}/heapster.cert"
-    - "{{ openshift_metrics_certs_dir }}/heapster.key"
+    - "{{ mktemp.stdout }}/heapster.cert"
+    - "{{ mktemp.stdout }}/heapster.key"
     - "{{ client_ca }}"
     vars:
-      custom_ca: "{{ openshift_metrics_certs_dir }}/heapster_client_ca.crt"
+      custom_ca: "{{ mktemp.stdout }}/heapster_client_ca.crt"
       default_ca: "{{ openshift.common.config_base }}/master/ca-bundle.crt"
       client_ca: "{{ custom_ca|exists|ternary(custom_ca, default_ca) }}"
   - name: generate heapster secret template
diff --git a/roles/openshift_metrics/tasks/import_jks_certs.yaml b/roles/openshift_metrics/tasks/import_jks_certs.yaml
index 16fd8d9f8..57ec70c79 100644
--- a/roles/openshift_metrics/tasks/import_jks_certs.yaml
+++ b/roles/openshift_metrics/tasks/import_jks_certs.yaml
@@ -1,37 +1,37 @@
 ---
-- stat: path="{{openshift_metrics_certs_dir}}/hawkular-cassandra.keystore"
+- stat: path="{{mktemp.stdout}}/hawkular-cassandra.keystore"
   register: cassandra_keystore
   check_mode: no
 
-- stat: path="{{openshift_metrics_certs_dir}}/hawkular-cassandra.truststore"
+- stat: path="{{mktemp.stdout}}/hawkular-cassandra.truststore"
   register: cassandra_truststore
   check_mode: no
 
-- stat: path="{{openshift_metrics_certs_dir}}/hawkular-metrics.keystore"
+- stat: path="{{mktemp.stdout}}/hawkular-metrics.keystore"
   register: metrics_keystore
   check_mode: no
 
-- stat: path="{{openshift_metrics_certs_dir}}/hawkular-metrics.truststore"
+- stat: path="{{mktemp.stdout}}/hawkular-metrics.truststore"
   register: metrics_truststore
   check_mode: no
 
-- stat: path="{{openshift_metrics_certs_dir}}/hawkular-jgroups.keystore"
+- stat: path="{{mktemp.stdout}}/hawkular-jgroups.keystore"
   register: jgroups_keystore
   check_mode: no
 
 - block:
-  - slurp: src={{ openshift_metrics_certs_dir }}/hawkular-metrics-keystore.pwd
+  - slurp: src={{ mktemp.stdout }}/hawkular-metrics-keystore.pwd
     register: metrics_keystore_password
 
-  - slurp: src={{ openshift_metrics_certs_dir }}/hawkular-cassandra-keystore.pwd
+  - slurp: src={{ mktemp.stdout }}/hawkular-cassandra-keystore.pwd
     register: cassandra_keystore_password
 
-  - slurp: src={{ openshift_metrics_certs_dir }}/hawkular-jgroups-keystore.pwd
+  - slurp: src={{ mktemp.stdout }}/hawkular-jgroups-keystore.pwd
     register: jgroups_keystore_password
 
   - fetch:
       dest: "{{local_tmp.stdout}}/"
-      src: "{{ openshift_metrics_certs_dir }}/{{item}}"
+      src: "{{ mktemp.stdout }}/{{item}}"
       flat: yes
     changed_when: False
     with_items:
@@ -52,7 +52,7 @@
     changed_when: False
 
   - copy:
-      dest: "{{openshift_metrics_certs_dir}}/"
+      dest: "{{mktemp.stdout}}/"
       src: "{{item}}"
     with_fileglob: "{{local_tmp.stdout}}/*.*store"
 
diff --git a/roles/openshift_metrics/tasks/install_hawkular.yaml b/roles/openshift_metrics/tasks/install_hawkular.yaml
index 1ba11efa8..6b37f85ab 100644
--- a/roles/openshift_metrics/tasks/install_hawkular.yaml
+++ b/roles/openshift_metrics/tasks/install_hawkular.yaml
@@ -17,7 +17,7 @@
   changed_when: false
 
 - name: read hawkular-metrics route destination ca certificate
-  slurp: src={{ openshift_metrics_certs_dir }}/ca.crt
+  slurp: src={{ mktemp.stdout }}/ca.crt
   register: metrics_route_dest_ca_cert
   changed_when: false
 
diff --git a/roles/openshift_metrics/tasks/main.yaml b/roles/openshift_metrics/tasks/main.yaml
index d03d4176b..1eebff3bf 100644
--- a/roles/openshift_metrics/tasks/main.yaml
+++ b/roles/openshift_metrics/tasks/main.yaml
@@ -9,6 +9,11 @@
   changed_when: False
   when: "{{ openshift_metrics_install_metrics | bool }}"
 
+- name: Create temp directory local on control node
+  local_action: command mktemp -d
+  register: local_tmp
+  changed_when: False
+
 - name: Copy the admin client config(s)
   command: >
      cp {{ openshift.common.config_base}}/master/admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig
@@ -17,3 +22,9 @@
   tags: metrics_init
 
 - include: "{{ (openshift_metrics_install_metrics | bool) | ternary('install_metrics.yaml','uninstall_metrics.yaml') }}"
+
+- name: Delete temp directory
+  local_action: file path=local_tmp.stdout state=absent
+  tags: metrics_cleanup
+  changed_when: False
+  check_mode: no
diff --git a/roles/openshift_metrics/tasks/pre_install.yaml b/roles/openshift_metrics/tasks/pre_install.yaml
index 262acd546..2e2013d40 100644
--- a/roles/openshift_metrics/tasks/pre_install.yaml
+++ b/roles/openshift_metrics/tasks/pre_install.yaml
@@ -12,12 +12,6 @@
   - openshift_metrics_cassandra_storage_type not in openshift_metrics_cassandra_storage_types
   - "not {{ openshift_metrics_heapster_standalone | bool }}"
 
-- name: create certificate output directory
-  file:
-    path: "{{ openshift_metrics_certs_dir }}"
-    state: directory
-    mode: 0700
-
 - name: list existing secrets
   command: >
     {{ openshift.common.client_binary }} -n {{ openshift_metrics_project }}
diff --git a/roles/openshift_metrics/tasks/setup_certificate.yaml b/roles/openshift_metrics/tasks/setup_certificate.yaml
index 5ca8f4462..199968579 100644
--- a/roles/openshift_metrics/tasks/setup_certificate.yaml
+++ b/roles/openshift_metrics/tasks/setup_certificate.yaml
@@ -3,50 +3,41 @@
   command: >
     {{ openshift.common.admin_binary }} ca create-server-cert
     --config={{ mktemp.stdout }}/admin.kubeconfig
-    --key='{{ openshift_metrics_certs_dir }}/{{ component }}.key'
-    --cert='{{ openshift_metrics_certs_dir }}/{{ component }}.crt'
+    --key='{{ mktemp.stdout }}/{{ component }}.key'
+    --cert='{{ mktemp.stdout }}/{{ component }}.crt'
     --hostnames='{{ hostnames }}'
-    --signer-cert='{{ openshift_metrics_certs_dir }}/ca.crt'
-    --signer-key='{{ openshift_metrics_certs_dir }}/ca.key'
-    --signer-serial='{{ openshift_metrics_certs_dir }}/ca.serial.txt'
-  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.key'|exists
+    --signer-cert='{{ mktemp.stdout }}/ca.crt'
+    --signer-key='{{ mktemp.stdout }}/ca.key'
+    --signer-serial='{{ mktemp.stdout }}/ca.serial.txt'
 
 - slurp: src={{item}}
   register: component_certs
   with_items:
-    - '{{ openshift_metrics_certs_dir | quote }}/{{ component|quote }}.key'
-    - '{{ openshift_metrics_certs_dir | quote }}/{{ component|quote }}.crt'
-  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'|exists
+    - '{{ mktemp.stdout | quote }}/{{ component|quote }}.key'
+    - '{{ mktemp.stdout | quote }}/{{ component|quote }}.crt'
 
 - name: generate {{ component }} certificate
   copy:
-    dest: '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'
+    dest: '{{ mktemp.stdout }}/{{ component }}.pem'
     content: "{{ component_certs.results | map(attribute='content') | map('b64decode') | join('')  }}"
-  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'|exists
 
 - name: generate random password for the {{ component }} keystore
   copy:
     content: "{{ 15 | oo_random_word }}"
-    dest: '{{ openshift_metrics_certs_dir }}/{{ component }}-keystore.pwd'
-  when: >
-    not '{{ openshift_metrics_certs_dir }}/{{ component }}-keystore.pwd'|exists
+    dest: '{{ mktemp.stdout }}/{{ component }}-keystore.pwd'
 
-- slurp: src={{ openshift_metrics_certs_dir | quote }}/{{ component|quote }}-keystore.pwd
+- slurp: src={{ mktemp.stdout | quote }}/{{ component|quote }}-keystore.pwd
   register: keystore_password
 
 - name: create the {{ component }} pkcs12 from the pem file
   command: >
     openssl pkcs12 -export
-    -in '{{ openshift_metrics_certs_dir }}/{{ component }}.pem'
-    -out '{{ openshift_metrics_certs_dir }}/{{ component }}.pkcs12'
+    -in '{{ mktemp.stdout }}/{{ component }}.pem'
+    -out '{{ mktemp.stdout }}/{{ component }}.pkcs12'
     -name '{{ component }}' -noiter -nomaciter
     -password 'pass:{{keystore_password.content | b64decode }}'
-  when: not '{{ openshift_metrics_certs_dir }}/{{ component }}.pkcs12'|exists
 
 - name: generate random password for the {{ component }} truststore
   copy:
     content: "{{ 15 | oo_random_word }}"
-    dest: '{{ openshift_metrics_certs_dir | quote }}/{{ component|quote }}-truststore.pwd'
-  when: >
-    not
-    '{{ openshift_metrics_certs_dir | quote }}/{{ component| quote  }}-truststore.pwd'|exists
+    dest: '{{ mktemp.stdout | quote }}/{{ component|quote }}-truststore.pwd'