16 files changed, 317 insertions, 32 deletions

diff --git a/roles/ansible_service_broker/defaults/main.yml b/roles/ansible_service_broker/defaults/main.yml
index dc05b03b5..34110ca99 100644
--- a/roles/ansible_service_broker/defaults/main.yml
+++ b/roles/ansible_service_broker/defaults/main.yml
@@ -14,3 +14,4 @@ ansible_service_broker_launch_apb_on_bind: false
 ansible_service_broker_image_pull_policy: IfNotPresent
 ansible_service_broker_sandbox_role: edit
 ansible_service_broker_auto_escalate: false
+ansible_service_broker_local_registry_whitelist: []
diff --git a/roles/ansible_service_broker/tasks/generate_certs.yml b/roles/ansible_service_broker/tasks/generate_certs.yml
new file mode 100644
index 000000000..50156a35c
--- /dev/null
+++ b/roles/ansible_service_broker/tasks/generate_certs.yml
@@ -0,0 +1,35 @@
+---
+
+- when: ansible_service_broker_certs_dir is undefined
+  block:
+  - name: Create ansible-service-broker cert directory
+    file:
+      path: "{{ openshift.common.config_base }}/ansible-service-broker"
+      state: directory
+      mode: 0755
+    check_mode: no
+
+  - set_fact:
+      ansible_service_broker_certs_dir: "{{ openshift.common.config_base }}/ansible-service-broker"
+
+  - name: Create self signing ca cert
+    command: 'openssl req -nodes -x509 -newkey rsa:4096 -keyout {{ ansible_service_broker_certs_dir }}/key.pem -out {{ ansible_service_broker_certs_dir }}/cert.pem -days 365 -subj "/CN=asb-etcd.openshift-ansible-service-broker.svc"'
+    args:
+      creates: '{{ ansible_service_broker_certs_dir }}/cert.pem'
+
+  - name: Create self signed client cert
+    command: '{{ item.cmd }}'
+    args:
+      creates: '{{ item.creates }}'
+    with_items:
+    - cmd: openssl genrsa -out {{ ansible_service_broker_certs_dir }}/client.key 2048
+      creates: '{{ ansible_service_broker_certs_dir }}/client.key'
+    - cmd: 'openssl req -new -key {{ ansible_service_broker_certs_dir }}/client.key -out {{ ansible_service_broker_certs_dir }}/client.csr -subj "/CN=client"'
+      creates: '{{ ansible_service_broker_certs_dir }}/client.csr'
+    - cmd: openssl x509 -req -in {{ ansible_service_broker_certs_dir }}/client.csr -CA {{ ansible_service_broker_certs_dir }}/cert.pem -CAkey {{ ansible_service_broker_certs_dir }}/key.pem -CAcreateserial -out {{ ansible_service_broker_certs_dir }}/client.pem -days 1024
+      creates: '{{ ansible_service_broker_certs_dir }}/client.pem'
+
+- set_fact:
+    etcd_ca_cert: "{{ lookup('file', '{{ ansible_service_broker_certs_dir }}/cert.pem') }}"
+    etcd_client_cert: "{{ lookup('file', '{{ ansible_service_broker_certs_dir }}/client.pem') }}"
+    etcd_client_key: "{{ lookup('file', '{{ ansible_service_broker_certs_dir }}/client.key') }}"
diff --git a/roles/ansible_service_broker/tasks/install.yml b/roles/ansible_service_broker/tasks/install.yml
index 808c24bbd..90a4418fb 100644
--- a/roles/ansible_service_broker/tasks/install.yml
+++ b/roles/ansible_service_broker/tasks/install.yml
@@ -32,6 +32,7 @@
 
 - include: validate_facts.yml
 
+- include: generate_certs.yml
 
 # Deployment of ansible-service-broker starts here
 - name: create openshift-ansible-service-broker project
@@ -68,6 +69,9 @@
       - apiGroups: ["authentication.k8s.io"]
         resources: ["tokenreviews"]
         verbs: ["create"]
+      - apiGroups: ["image.openshift.io", ""]
+        resources: ["images"]
+        verbs: ["get", "list"]
 
 - name: Create asb-access cluster role
   oc_clusterrole:
@@ -116,6 +120,24 @@
             kubernetes.io/service-account.name: asb-client
         type: kubernetes.io/service-account-token
 
+- name: Create etcd-auth secret
+  oc_secret:
+    name: etcd-auth-secret
+    namespace: openshift-ansible-service-broker
+    contents:
+      - path: ca.crt
+        data: '{{ etcd_ca_cert }}'
+
+- name: Create broker-etcd-auth secret
+  oc_secret:
+    name: broker-etcd-auth-secret
+    namespace: openshift-ansible-service-broker
+    contents:
+      - path: client.crt
+        data: '{{ etcd_client_cert }}'
+      - path: client.key
+        data: '{{ etcd_client_key }}'
+
 - oc_secret:
     state: list
     namespace: openshift-ansible-service-broker
@@ -156,6 +178,34 @@
             app: openshift-ansible-service-broker
             service: asb
 
+- name: create asb-etcd service
+  oc_obj:
+    name: asb-etcd
+    namespace: openshift-ansible-service-broker
+    state: present
+    kind: Service
+    content:
+      path: /tmp/asbetcdsvcout
+      data:
+        apiVersion: v1
+        kind: Service
+        metadata:
+          name: asb-etcd
+          labels:
+            app: etcd
+            service: asb-etcd
+          annotations:
+            service.alpha.openshift.io/serving-cert-secret-name: etcd-tls
+        spec:
+          ports:
+            - name: port-2379
+              port: 2379
+              targetPort: 2379
+              protocol: TCP
+          selector:
+            app: etcd
+            service: asb-etcd
+
 - name: create route for ansible-service-broker service
   oc_route:
     name: asb-1338
@@ -227,6 +277,8 @@
                       mountPath: /etc/ansible-service-broker
                     - name: asb-tls
                       mountPath: /etc/tls/private
+                    - name: asb-etcd-auth
+                      mountPath: /var/run/asb-etcd-auth
                   ports:
                     - containerPort: 1338
                       protocol: TCP
@@ -249,7 +301,50 @@
                       scheme: HTTPS
                     initialDelaySeconds: 15
                     timeoutSeconds: 1
+              volumes:
+                - name: config-volume
+                  configMap:
+                    name: broker-config
+                    items:
+                      - key: broker-config
+                        path: config.yaml
+                - name: asb-tls
+                  secret:
+                    secretName: asb-tls
+                - name: asb-etcd-auth
+                  secret:
+                    secretName: broker-etcd-auth-secret
 
+- name: Create asb-etcd deployment config
+  oc_obj:
+    name: etcd
+    namespace: openshift-ansible-service-broker
+    state: present
+    kind: DeploymentConfig
+    content:
+      path: /tmp/dcout
+      data:
+        apiVersion: v1
+        kind: DeploymentConfig
+        metadata:
+          name: asb-etcd
+          labels:
+            app: etcd
+            service: asb-etcd
+        spec:
+          replicas: 1
+          selector:
+            app: etcd
+          strategy:
+            type: Rolling
+          template:
+            metadata:
+              labels:
+                app: etcd
+                service: asb-etcd
+            spec:
+              serviceAccount: asb
+              containers:
                 - image: "{{ ansible_service_broker_etcd_image }}"
                   name: etcd
                   imagePullPolicy: IfNotPresent
@@ -258,8 +353,12 @@
                   args:
                     - "{{ ansible_service_broker_etcd_image_etcd_path }}"
                     - "--data-dir=/data"
-                    - "--listen-client-urls=http://0.0.0.0:2379"
-                    - "--advertise-client-urls=http://0.0.0.0:2379"
+                    - "--listen-client-urls=https://0.0.0.0:2379"
+                    - "--advertise-client-urls=https://0.0.0.0:2379"
+                    - "--client-cert-auth"
+                    - "--trusted-ca-file=/var/run/etcd-auth-secret/ca.crt"
+                    - "--cert-file=/etc/tls/private/tls.crt"
+                    - "--key-file=/etc/tls/private/tls.key"
                   ports:
                     - containerPort: 2379
                       protocol: TCP
@@ -267,21 +366,22 @@
                     - name: ETCDCTL_API
                       value: "3"
                   volumeMounts:
-                    - mountPath: /data
-                      name: etcd
+                    - name: etcd
+                      mountPath: /data
+                    - name: etcd-tls
+                      mountPath: /etc/tls/private
+                    - name: etcd-auth
+                      mountPath: /var/run/etcd-auth-secret
               volumes:
                 - name: etcd
                   persistentVolumeClaim:
                     claimName: etcd
-                - name: config-volume
-                  configMap:
-                    name: broker-config
-                    items:
-                      - key: broker-config
-                        path: config.yaml
-                - name: asb-tls
+                - name: etcd-tls
                   secret:
-                    secretName: asb-tls
+                    secretName: etcd-tls
+                - name: etcd-auth
+                  secret:
+                    secretName: etcd-auth-secret
 
 
 # TODO: saw a oc_configmap in the library, but didn't understand how to get it to do the following:
@@ -307,14 +407,18 @@
               - type: {{ ansible_service_broker_registry_type }}
                 name: {{ ansible_service_broker_registry_name }}
                 url:  {{ ansible_service_broker_registry_url }}
-                user: {{ ansible_service_broker_registry_user }}
-                pass: {{ ansible_service_broker_registry_password }}
                 org:  {{ ansible_service_broker_registry_organization }}
                 tag:  {{ ansible_service_broker_registry_tag }}
                 white_list: {{ ansible_service_broker_registry_whitelist }}
+              - type: local_registry
+                namespaces: ['openshift']
+                white_list: {{ ansible_service_broker_local_registry_whitelist }}
             dao:
-              etcd_host: 0.0.0.0
+              etcd_host: asb-etcd.openshift-ansible-service-broker.svc
               etcd_port: 2379
+              etcd_ca_file: /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+              etcd_client_cert: /var/run/asb-etcd-auth/client.crt
+              etcd_client_key: /var/run/asb-etcd-auth/client.key
             log:
               stdout: true
               level: {{ ansible_service_broker_log_level }}
@@ -339,6 +443,15 @@
                 - type: basic
                   enabled: false
 
+- oc_secret:
+    name: asb-registry-auth
+    namespace: openshift-ansible-service-broker
+    state: present
+    contents:
+      - path: username
+        data: "{{ ansible_service_broker_registry_user }}"
+      - path: password
+        data: "{{ ansible_service_broker_registry_password }}"
 
 - name: Create the Broker resource in the catalog
   oc_obj:
diff --git a/roles/ansible_service_broker/tasks/remove.yml b/roles/ansible_service_broker/tasks/remove.yml
index 51b86fb26..a1ac740e0 100644
--- a/roles/ansible_service_broker/tasks/remove.yml
+++ b/roles/ansible_service_broker/tasks/remove.yml
@@ -46,18 +46,42 @@
     resource_name: asb-access
     user: "system:serviceaccount:openshift-ansible-service-broker:asb-client"
 
+- name: remove asb-registry auth secret
+  oc_secret:
+    state: absent
+    name: asb-registry-auth
+    namespace: openshift-ansible-service-broker
+
 - name: remove asb-client token secret
   oc_secret:
     state: absent
     name: asb-client
     namespace: openshift-ansible-service-broker
 
+- name: Remove etcd-auth secret
+  oc_secret:
+    state: absent
+    name: etcd-auth-secret
+    namespace: openshift-ansible-service-broker
+
+- name: Remove broker-etcd-auth secret
+  oc_secret:
+    state: absent
+    name: broker-etcd-auth-secret
+    namespace: openshift-ansible-service-broker
+
 - name: remove ansible-service-broker service
   oc_service:
     name: asb
     namespace: openshift-ansible-service-broker
     state: absent
 
+- name: remove asb-etcd service
+  oc_service:
+    state: absent
+    name: asb-etcd
+    namespace: openshift-ansible-service-broker
+
 - name: remove etcd service
   oc_service:
     name: etcd
@@ -83,6 +107,14 @@
     kind: DeploymentConfig
     state: absent
 
+- name: remove Ansible Service Broker etcd deployment config
+  oc_obj:
+    name: asb-etcd
+    namespace: openshift-ansible-service-broker
+    kind: DeploymentConfig
+    state: absent
+
+
 - name: remove secret for broker auth
   oc_obj:
     name: asb-client
diff --git a/roles/ansible_service_broker/tasks/validate_facts.yml b/roles/ansible_service_broker/tasks/validate_facts.yml
index 604d24e1d..a2345551b 100644
--- a/roles/ansible_service_broker/tasks/validate_facts.yml
+++ b/roles/ansible_service_broker/tasks/validate_facts.yml
@@ -1,11 +1,9 @@
 ---
 - name: validate Dockerhub registry settings
-  fail: msg="To use the dockerhub registry, you must provide the ansible_service_broker_registry_user. ansible_service_broker_registry_password, and ansible_service_broker_registry_organization parameters"
+  fail: msg="To use the dockerhub registry, you must provide the ansible_service_broker_registry_organization"
   when:
     - ansible_service_broker_registry_type == 'dockerhub'
-    - not (ansible_service_broker_registry_user and
-        ansible_service_broker_registry_password and
-        ansible_service_broker_registry_organization)
+    - not ansible_service_broker_registry_organization
 
 
 - name: validate RHCC registry settings
diff --git a/roles/ansible_service_broker/vars/default_images.yml b/roles/ansible_service_broker/vars/default_images.yml
index 8438e993f..248e0363d 100644
--- a/roles/ansible_service_broker/vars/default_images.yml
+++ b/roles/ansible_service_broker/vars/default_images.yml
@@ -12,6 +12,6 @@ __ansible_service_broker_registry_name: dh
 __ansible_service_broker_registry_url: null
 __ansible_service_broker_registry_user: null
 __ansible_service_broker_registry_password: null
-__ansible_service_broker_registry_organization: null
+__ansible_service_broker_registry_organization: ansibleplaybookbundle
 __ansible_service_broker_registry_tag: latest
 __ansible_service_broker_registry_whitelist: []
diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml
index f6f2bd77e..c086c28df 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/docker/defaults/main.yml
@@ -33,3 +33,6 @@ r_crio_os_firewall_allow:
 
 
 openshift_docker_is_node_or_master: "{{ True if inventory_hostname in (groups['oo_masters_to_config']|default([])) or inventory_hostname in (groups['oo_nodes_to_config']|default([])) else False | bool }}"
+
+docker_alt_storage_path: /var/lib/containers/docker
+docker_default_storage_path: /var/lib/docker
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 1539af53f..3c814d8d8 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -25,6 +25,15 @@
     - not l_use_system_container
     - not l_use_crio_only
 
+- name: Ensure /var/lib/containers exists
+  file:
+    path: /var/lib/containers
+    state: directory
+
+- name: Fix SELinux Permissions on /var/lib/containers
+  command: "restorecon -R /var/lib/containers/"
+  changed_when: false
+
 - name: Use System Container Docker if Requested
   include: systemcontainer_docker.yml
   when:
@@ -36,3 +45,48 @@
   when:
     - l_use_crio
     - openshift_docker_is_node_or_master | bool
+
+- name: stat the docker data dir
+  stat:
+    path: "{{ docker_default_storage_path }}"
+  register: dockerstat
+
+- when:
+    - l_use_crio
+    - dockerstat.stat.islink is defined and not (dockerstat.stat.islink | bool)
+  block:
+    - name: stop the current running docker
+      systemd:
+        state: stopped
+        name: "{{ openshift.docker.service_name }}"
+
+    - name: "Ensure {{ docker_alt_storage_path }} exists"
+      file:
+        path: "{{ docker_alt_storage_path }}"
+        state: directory
+
+    - name: "Set the selinux context on {{ docker_alt_storage_path }}"
+      command: "semanage fcontext -a -e {{ docker_default_storage_path }} {{ docker_alt_storage_path }}"
+      register: results
+      failed_when:
+        - results.rc == 1
+        - "'already exists' not in results.stderr"
+
+    - name: "restorecon the {{ docker_alt_storage_path }}"
+      command: "restorecon -r {{ docker_alt_storage_path }}"
+
+    - name: Remove the old docker location
+      file:
+        state: absent
+        path: "{{ docker_default_storage_path }}"
+
+    - name: Setup the link
+      file:
+        state: link
+        src: "{{ docker_alt_storage_path }}"
+        path: "{{ docker_default_storage_path }}"
+
+    - name: start docker
+      systemd:
+        state: started
+        name: "{{ openshift.docker.service_name }}"
diff --git a/roles/docker/tasks/systemcontainer_crio.yml b/roles/docker/tasks/systemcontainer_crio.yml
index 67ede0d21..1e2d64293 100644
--- a/roles/docker/tasks/systemcontainer_crio.yml
+++ b/roles/docker/tasks/systemcontainer_crio.yml
@@ -170,10 +170,6 @@
     dest: /etc/cni/net.d/openshift-sdn.conf
     src: 80-openshift-sdn.conf.j2
 
-- name: Fix SELinux Permissions on /var/lib/containers
-  command: "restorecon -R /var/lib/containers/"
-  changed_when: false
-
 - name: Start the CRI-O service
   systemd:
     name: "cri-o"
diff --git a/roles/nuage_master/handlers/main.yaml b/roles/nuage_master/handlers/main.yaml
index 21da6b953..410b739e9 100644
--- a/roles/nuage_master/handlers/main.yaml
+++ b/roles/nuage_master/handlers/main.yaml
@@ -7,8 +7,13 @@
     openshift.master.cluster_method == 'native'
 
 # TODO: need to fix up ignore_errors here
+# We retry the controllers because the API may not be 100% initialized yet.
 - name: restart master controllers
-  systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  command: "systemctl restart {{ openshift.common.service_type }}-master-controllers"
+  retries: 3
+  delay: 5
+  register: result
+  until: result.rc == 0
   when: >
     (openshift_master_ha | bool) and
     (not master_controllers_service_status_changed | default(false)) and
diff --git a/roles/openshift_ca/tasks/main.yml b/roles/openshift_ca/tasks/main.yml
index fad1ff5de..587526d07 100644
--- a/roles/openshift_ca/tasks/main.yml
+++ b/roles/openshift_ca/tasks/main.yml
@@ -106,6 +106,36 @@
   delegate_to: "{{ openshift_ca_host }}"
   run_once: true
 
+# Create client-ca-bundle.crt containing old and new OpenShift CA
+# certificates. This bundle will be used when rolling the OpenShift CA
+# certificate.
+- name: Create client-ca-bundle.crt
+  block:
+  - command: mktemp -d /tmp/openshift-ansible-XXXXXX
+    register: openshift_ca_clientconfig_tmpdir
+    delegate_to: "{{ openshift_ca_host }}"
+  - copy:
+      src: "{{ item }}"
+      dest: "{{ openshift_ca_clientconfig_tmpdir.stdout }}/"
+      remote_src: true
+    with_items: "{{ g_master_legacy_ca_result.files | default([]) | oo_collect('path') }}"
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+  - copy:
+      src: "{{ openshift_ca_config_dir }}/ca.crt"
+      dest: "{{ openshift_ca_clientconfig_tmpdir.stdout }}/"
+      remote_src: true
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+  - assemble:
+      src: "{{ openshift_ca_clientconfig_tmpdir.stdout }}"
+      dest: "{{ openshift_ca_config_dir }}/client-ca-bundle.crt"
+      mode: 0644
+      owner: root
+      group: root
+    delegate_to: "{{ openshift_ca_host }}"
+    run_once: true
+
 - name: Test local loopback context
   command: >
     {{ hostvars[openshift_ca_host].openshift.common.client_binary }} config view
diff --git a/roles/openshift_hosted_metrics/handlers/main.yml b/roles/openshift_hosted_metrics/handlers/main.yml
index ce7688581..88b893448 100644
--- a/roles/openshift_hosted_metrics/handlers/main.yml
+++ b/roles/openshift_hosted_metrics/handlers/main.yml
@@ -4,8 +4,13 @@
   when: (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
   notify: Verify API Server
 
+# We retry the controllers because the API may not be 100% initialized yet.
 - name: restart master controllers
-  systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  command: "systemctl restart {{ openshift.common.service_type }}-master-controllers"
+  retries: 3
+  delay: 5
+  register: result
+  until: result.rc == 0
   when: (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
 
 - name: Verify API Server
diff --git a/roles/openshift_logging/handlers/main.yml b/roles/openshift_logging/handlers/main.yml
index ce7688581..acc838bd1 100644
--- a/roles/openshift_logging/handlers/main.yml
+++ b/roles/openshift_logging/handlers/main.yml
@@ -4,8 +4,13 @@
   when: (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
   notify: Verify API Server
 
+# We retry the controllers because the API may not be 100% initialized yet.
 - name: restart master controllers
-  systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  command: "{{ openshift.common.service_type }}-master-controllers"
+  retries: 3
+  delay: 5
+  register: result
+  until: result.rc == 0
   when: (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
 
 - name: Verify API Server
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index d5094c2c9..f88c4a7dc 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -9,10 +9,13 @@
   notify:
   - Verify API Server
 
+# We retry the controllers because the API may not be 100% initialized yet.
 - name: restart master controllers
-  systemd:
-    name: "{{ openshift.common.service_type }}-master-controllers"
-    state: restarted
+  command: "systemctl restart {{ openshift.common.service_type }}-master-controllers"
+  retries: 3
+  delay: 5
+  register: result
+  until: result.rc == 0
   when:
   - not (master_controllers_service_status_changed | default(false) | bool)
   - openshift.master.cluster_method == 'native'
diff --git a/roles/openshift_master_facts/filter_plugins/openshift_master.py b/roles/openshift_master_facts/filter_plugins/openshift_master.py
index 69fecc7d2..1bea66c26 100644
--- a/roles/openshift_master_facts/filter_plugins/openshift_master.py
+++ b/roles/openshift_master_facts/filter_plugins/openshift_master.py
@@ -508,7 +508,7 @@ class FilterModule(object):
                  'master.kubelet-client.crt',
                  'master.kubelet-client.key']
         if bool(include_ca):
-            certs += ['ca.crt', 'ca.key', 'ca-bundle.crt']
+            certs += ['ca.crt', 'ca.key', 'ca-bundle.crt', 'client-ca-bundle.crt']
         if bool(include_keys):
             certs += ['serviceaccounts.private.key',
                       'serviceaccounts.public.key']
diff --git a/roles/openshift_metrics/handlers/main.yml b/roles/openshift_metrics/handlers/main.yml
index ce7688581..88b893448 100644
--- a/roles/openshift_metrics/handlers/main.yml
+++ b/roles/openshift_metrics/handlers/main.yml
@@ -4,8 +4,13 @@
   when: (not (master_api_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
   notify: Verify API Server
 
+# We retry the controllers because the API may not be 100% initialized yet.
 - name: restart master controllers
-  systemd: name={{ openshift.common.service_type }}-master-controllers state=restarted
+  command: "systemctl restart {{ openshift.common.service_type }}-master-controllers"
+  retries: 3
+  delay: 5
+  register: result
+  until: result.rc == 0
   when: (not (master_controllers_service_status_changed | default(false) | bool)) and openshift.master.cluster_method == 'native'
 
 - name: Verify API Server