Refactor openshift_hosted's docker-registry route setup

We have identified an issue where a docker-registry service set up
as 'reencrypt' with a provided certificate and a self-signed certificate
on the pod does not authorize users to push images.
If the docker-registry service is set up as 'passthrough' with the
same provided certificate, everything works.

In light of this, this commit essentially adds support for configuring
provided certificates with a passthrough route while maintaining backwards
compatibility with the other use cases.
The default remains 'passthrough' with self-generated certificates.

Other miscellaneous changes include:
- Move fact setup that were only used in secure.yml there
- Omit the hostname for the route if there are none to configure,
  oc_route takes care of handling the default
- Replace hardcoded /etc/origin/master by openshift_master_config_dir

4 files changed, 138 insertions, 49 deletions

diff --git a/roles/openshift_hosted/tasks/registry/registry.yml b/roles/openshift_hosted/tasks/registry/registry.yml
index b946ec8ca..45777c26b 100644
--- a/roles/openshift_hosted/tasks/registry/registry.yml
+++ b/roles/openshift_hosted/tasks/registry/registry.yml
@@ -40,9 +40,6 @@
     openshift_hosted_registry_images: "{{ openshift.hosted.registry.registryurl | default('openshift3/ose-${component}:${version}')}}"
     openshift_hosted_registry_volumes: []
     openshift_hosted_registry_env_vars: {}
-    openshift_hosted_registry_routecertificates: "{{ ('routecertificates' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routecertificates, {}) }}"
-    openshift_hosted_registry_routehost: "{{ ('routehost' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routehost, False) }}"
-    openshift_hosted_registry_routetermination: "{{ ('routetermination' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routetermination, 'passthrough') }}"
     openshift_hosted_registry_edits:
     # These edits are being specified only to prevent 'changed' on rerun
     - key: spec.strategy.rollingParams
diff --git a/roles/openshift_hosted/tasks/registry/secure.yml b/roles/openshift_hosted/tasks/registry/secure.yml
index 29c164f52..a18e6eea9 100644
--- a/roles/openshift_hosted/tasks/registry/secure.yml
+++ b/roles/openshift_hosted/tasks/registry/secure.yml
@@ -1,65 +1,68 @@
 ---
-- name: Set fact docker_registry_route_hostname
+- name: Configure facts for docker-registry
   set_fact:
-    docker_registry_route_hostname: "{{ 'docker-registry-default.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}"
+    openshift_hosted_registry_routecertificates: "{{ ('routecertificates' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routecertificates, {}) }}"
+    openshift_hosted_registry_routehost: "{{ ('routehost' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routehost, False) }}"
+    openshift_hosted_registry_routetermination: "{{ ('routetermination' in openshift.hosted.registry.keys()) | ternary(openshift.hosted.registry.routetermination, 'passthrough') }}"
 
-- name: Get the certificate contents for registry
-  copy:
-    backup: True
-    dest: "/etc/origin/master/named_certificates/{{ item.value | basename }}"
-    src: "{{ item.value }}"
-  when: item.key in ['certfile', 'keyfile', 'cafile'] and item.value
-  with_dict: "{{ openshift_hosted_registry_routecertificates }}"
+- name: Include reencrypt route configuration
+  include: secure/reencrypt.yml
+  static: no
+  when: openshift_hosted_registry_routetermination == 'reencrypt'
 
-# When certificates are defined we will create the reencrypt
-# docker-registry route
-- name: Create a reencrypt route for docker-registry
-  oc_route:
-    name: docker-registry
-    namespace: "{{ openshift_hosted_registry_namespace }}"
-    service_name: docker-registry
-    tls_termination: "{{ openshift_hosted_registry_routetermination }}"
-    host: "{{ openshift_hosted_registry_routehost | default(docker_registry_route_hostname) }}"
-    cert_path: "/etc/origin/master/named_certificates/{{ openshift_hosted_registry_routecertificates['certfile'] | basename }}"
-    key_path: "/etc/origin/master/named_certificates/{{ openshift_hosted_registry_routecertificates['keyfile'] | basename }}"
-    cacert_path: "/etc/origin/master/named_certificates/{{ openshift_hosted_registry_routecertificates['cafile'] | basename }}"
-    dest_cacert_path: /etc/origin/master/ca.crt
-  when:
-  - "'cafile' in openshift_hosted_registry_routecertificates"
-  - "'certfile' in openshift_hosted_registry_routecertificates"
-  - "'keyfile' in openshift_hosted_registry_routecertificates"
+- name: Include passthrough route configuration
+  include: secure/passthrough.yml
+  static: no
+  when: openshift_hosted_registry_routetermination == 'passthrough'
 
-# When routetermination is passthrough we will create the route
-- name: Create passthrough route for docker-registry
+- name: Fetch the docker-registry route
   oc_route:
     name: docker-registry
-    namespace: "{{ openshift_hosted_registry_namespace }}"
-    service_name: docker-registry
-    tls_termination: "{{ openshift_hosted_registry_routetermination }}"
-    host: "{{ openshift_hosted_registry_routehost | ternary(openshift_hosted_registry_routehost, docker_registry_route_hostname) }}"
-  when: openshift_hosted_registry_routetermination == 'passthrough'
+    namespace: default
+    state: list
+  register: docker_registry_route
 
-- name: Retrieve registry service IP
+- name: Retrieve registry service for the clusterip
   oc_service:
     namespace: "{{ openshift_hosted_registry_namespace }}"
     name: docker-registry
     state: list
-  register: docker_registry_service_ip
+  register: docker_registry_service
 
-- name: Create registry certificates
+- name: Generate self-signed docker-registry certificates
   oc_adm_ca_server_cert:
     signer_cert: "{{ openshift_master_config_dir }}/ca.crt"
     signer_key: "{{ openshift_master_config_dir }}/ca.key"
     signer_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"
     hostnames:
-    - "{{ docker_registry_service_ip.results.clusterip }}"
-    - "{{ openshift_hosted_registry_name }}.default.svc"
-    - "{{ openshift_hosted_registry_name }}.default.svc.{{ openshift.common.dns_domain }}"
-    - "{{ docker_registry_route_hostname }}"
-    cert: "{{ openshift_master_config_dir }}/registry.crt"
-    key: "{{ openshift_master_config_dir }}/registry.key"
+    - "{{ docker_registry_service.results.clusterip }}"
+    - "{{ docker_registry_route.results[0].spec.host }}"
+    cert: "{{ docker_registry_cert_path }}"
+    key: "{{ docker_registry_key_path }}"
     expire_days: "{{ openshift_hosted_registry_cert_expire_days if openshift_version | oo_version_gte_3_5_or_1_5(openshift.common.deployment_type) | bool else omit }}"
-  register: server_cert_out
+  register: registry_self_cert
+  when: docker_registry_self_signed
+
+# Setting up REGISTRY_HTTP_TLS_CLIENTCAS as the cacert doesn't seem to work.
+# If we need to set up a cacert, bundle it with the cert.
+- when: docker_registry_cacert_path is defined
+  block:
+  - name: Retrieve certificate files to generate certificate bundle
+    slurp:
+      src: "{{ item }}"
+    with_items:
+    - "{{ docker_registry_cert_path }}"
+    - "{{ docker_registry_cacert_path }}"
+    register: certificate_files
+
+  - name: Generate certificate bundle
+    copy:
+      content: "{{ certificate_files.results | map(attribute='content') | map('b64decode') | join('') }}"
+      dest: "{{ openshift_master_config_dir }}/named_certificates/docker-registry.pem"
+
+  - name: Reset the certificate path to use the bundle
+    set_fact:
+      docker_registry_cert_path: "{{ openshift_master_config_dir }}/named_certificates/docker-registry.pem"
 
 - name: Create the secret for the registry certificates
   oc_secret:
@@ -67,9 +70,9 @@
     namespace: "{{ openshift_hosted_registry_namespace }}"
     files:
     - name: registry.crt
-      path: "{{ openshift_master_config_dir }}/registry.crt"
+      path: "{{ docker_registry_cert_path }}"
     - name: registry.key
-      path: "{{ openshift_master_config_dir }}/registry.key"
+      path: "{{ docker_registry_key_path }}"
   register: create_registry_certificates_secret_out
 
 - name: Add the secret to the registry's pod service accounts
@@ -99,9 +102,15 @@
       value: HTTPS
       action: put
 
+- name: Detect if there has been certificate changes
+  set_fact:
+    registry_cert_changed: true
+  when: ( registry_self_cert is defined and registry_self_cert.changed ) or
+        create_registry_certificates_secret_out.changed
+
 - name: Update openshift_hosted facts with secure registry variables
   set_fact:
     openshift_hosted_registry_volumes: "{{ openshift_hosted_registry_volumes | union(registry_secure_volume_mounts) }}"
     openshift_hosted_registry_env_vars: "{{ openshift_hosted_registry_env_vars | combine(registry_secure_env_vars) }}"
     openshift_hosted_registry_edits: "{{ openshift_hosted_registry_edits | union(registry_secure_edits) }}"
-    openshift_hosted_registry_force: "{{ openshift_hosted_registry_force | union([server_cert_out.changed]) | union([create_registry_certificates_secret_out.changed]) }}"
+    openshift_hosted_registry_force: "{{ openshift_hosted_registry_force | union([registry_cert_changed | default(false)]) }}"
diff --git a/roles/openshift_hosted/tasks/registry/secure/passthrough.yml b/roles/openshift_hosted/tasks/registry/secure/passthrough.yml
new file mode 100644
index 000000000..5b44fda10
--- /dev/null
+++ b/roles/openshift_hosted/tasks/registry/secure/passthrough.yml
@@ -0,0 +1,45 @@
+---
+# Generate a self-signed certificate when there is no user-supplied certificate
+- name: Configure self-signed certificate file paths
+  set_fact:
+    docker_registry_cert_path: "{{ openshift_master_config_dir }}/registry.crt"
+    docker_registry_key_path: "{{ openshift_master_config_dir }}/registry.key"
+    docker_registry_cacert_path: "{{ openshift_master_config_dir }}/ca.crt"
+    docker_registry_self_signed: true
+  when:
+  - "'certfile' not in openshift_hosted_registry_routecertificates"
+  - "'keyfile' not in openshift_hosted_registry_routecertificates"
+
+# Retrieve user supplied certificate files if they are provided
+- when:
+  - "'certfile' in openshift_hosted_registry_routecertificates"
+  - "'keyfile' in openshift_hosted_registry_routecertificates"
+  block:
+  - name: Configure provided certificate file paths
+    set_fact:
+      docker_registry_cert_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['certfile'] | basename }}"
+      docker_registry_key_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['keyfile'] | basename }}"
+      docker_registry_self_signed: false
+
+  # Since we end up bundling the cert, cacert and key in a .pem file, the 'cafile'
+  # is optional
+  - name: Configure provided ca certificate file path
+    set_fact:
+      docker_registry_cacert_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['cafile'] | basename }}"
+    when: "'cafile' in openshift_hosted_registry_routecertificates"
+
+  - name: Retrieve provided certificate files
+    copy:
+      backup: True
+      dest: "{{ openshift_master_config_dir }}/named_certificates/{{ item.value | basename }}"
+      src: "{{ item.value }}"
+    when: item.key in ['certfile', 'keyfile', 'cafile'] and item.value
+    with_dict: "{{ openshift_hosted_registry_routecertificates }}"
+
+- name: Configure a passthrough route for docker-registry
+  oc_route:
+    name: docker-registry
+    namespace: "{{ openshift_hosted_registry_namespace }}"
+    service_name: docker-registry
+    tls_termination: "{{ openshift_hosted_registry_routetermination }}"
+    host: "{{ openshift_hosted_registry_routehost | default(omit, true) }}"
diff --git a/roles/openshift_hosted/tasks/registry/secure/reencrypt.yml b/roles/openshift_hosted/tasks/registry/secure/reencrypt.yml
new file mode 100644
index 000000000..48e5b0fba
--- /dev/null
+++ b/roles/openshift_hosted/tasks/registry/secure/reencrypt.yml
@@ -0,0 +1,38 @@
+---
+- name: Validate route termination configuration
+  fail:
+    msg: >
+     When 'openshift_hosted_registry_routetermination' is 'reencrypt', you must
+     provide certificate files with 'openshift_hosted_registry_routecertificates'
+  when: ('certfile' not in openshift_hosted_registry_routecertificates) or
+        ('keyfile' not in openshift_hosted_registry_routecertificates) or
+        ('cafile' not in openshift_hosted_registry_routecertificates)
+
+- name: Configure self-signed certificate file paths
+  set_fact:
+    docker_registry_cert_path: "{{ openshift_master_config_dir }}/registry.crt"
+    docker_registry_key_path: "{{ openshift_master_config_dir }}/registry.key"
+    docker_registry_cacert_path: "{{ openshift_master_config_dir }}/ca.crt"
+    docker_registry_self_signed: true
+
+- name: Retrieve provided certificate files
+  copy:
+    backup: True
+    dest: "{{ openshift_master_config_dir }}/named_certificates/{{ item.value | basename }}"
+    src: "{{ item.value }}"
+  when: item.key in ['certfile', 'keyfile', 'cafile'] and item.value
+  with_dict: "{{ openshift_hosted_registry_routecertificates }}"
+
+# Encrypt with the provided certificate and provide the dest_cacert for the
+# self-signed certificate at the endpoint
+- name: Configure a reencrypt route for docker-registry
+  oc_route:
+    name: docker-registry
+    namespace: "{{ openshift_hosted_registry_namespace }}"
+    service_name: docker-registry
+    tls_termination: "{{ openshift_hosted_registry_routetermination }}"
+    host: "{{ openshift_hosted_registry_routehost | default(omit, true) }}"
+    cert_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['certfile'] | basename }}"
+    key_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['keyfile'] | basename }}"
+    cacert_path: "{{ openshift_master_config_dir }}/named_certificates/{{ openshift_hosted_registry_routecertificates['cafile'] | basename }}"
+    dest_cacert_path: "{{ openshift_master_config_dir }}/ca.crt"