Handling GlusterFS storage security in OpenShift containers

19 files changed, 289 insertions, 28 deletions

diff --git a/playbooks/ands-setup-vm.yml b/playbooks/ands-setup-vm.yml
deleted file mode 100644
index d97916d..0000000
--- a/playbooks/ands-setup-vm.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-- name: Common setup procedures
-  hosts: vagrant
-  roles:
-    - role: ands_vagrant_vm
-
diff --git a/playbooks/openshift-setup-projects.yml b/playbooks/openshift-setup-projects.yml
index a8af9c1..cc36498 100644
--- a/playbooks/openshift-setup-projects.yml
+++ b/playbooks/openshift-setup-projects.yml
@@ -15,6 +15,7 @@
   hosts: masters
   roles:
     - { role: ands_openshift, subrole: users }
+    - { role: ands_openshift, subrole: security }
     - { role: ands_openshift, subrole: storage }
     - { role: ands_kaas }
   vars:
diff --git a/playbooks/openshift-setup-security.yml b/playbooks/openshift-setup-security.yml
new file mode 100644
index 0000000..6c85602
--- /dev/null
+++ b/playbooks/openshift-setup-security.yml
@@ -0,0 +1,24 @@
+- name: Configure users
+  hosts: masters
+  roles:
+    - { role: ands_facts }
+
+
+- name: Temporary provision /etc/hosts with Masters IP.
+  hosts: nodes:!masters
+  tasks:
+    - lineinfile: dest="/etc/hosts" line="{{ ands_openshift_network | ipaddr(node_id) | ipaddr('address') }} {{ ands_openshift_lb }}" regexp=".*{{ ands_openshift_lb }}$" state="present"
+      when: (ands_provision_without_dns | default(false))
+  vars:
+    node_id: "{{ hostvars[groups['masters'][0]]['ands_host_id'] }}"
+
+- name: Configure security
+  hosts: masters
+  roles:
+    - { role: ands_openshift, subrole: security }
+
+- name: Remove temporary entries in /etc/hosts
+  hosts: nodes:!masters
+  tasks:
+    - lineinfile: dest="/etc/hosts" regexp=".*{{ ands_openshift_lb }}$" state="absent"
+      when: (ands_provision_without_dns | default(false))
diff --git a/roles/ands_kaas/tasks/file.yml b/roles/ands_kaas/tasks/file.yml
index 9a36e74..479ec68 100644
--- a/roles/ands_kaas/tasks/file.yml
+++ b/roles/ands_kaas/tasks/file.yml
@@ -1,9 +1,23 @@
 ---
+- name: Set group
+  set_fact: group="{{ file.group | default(kaas_project_config.file_group | default(ands_default_file_group)) }}"
+
+- name : Resolve project groups
+  set_fact: group="{{ (kaas_project_config.gids | default(ands_openshift_gids))[group].id }}"
+  when: group in ( kaas_project_config.gids | default(ands_openshift_gids) )
+
+- name: Set owner
+  set_fact: owner="{{ file.owner | default(kaas_project_config.file_owner | default(ands_default_file_owner)) }}"
+
+- name : Resolve project uids
+  set_fact: owner="{{ (kaas_project_config.uids | default(ands_openshift_uids) )[owner].id }}"
+  when: owner in ( kaas_project_config.uids | default(ands_openshift_uids) )
+
 - name: "Setting up files in {{ path }}"
   file: 
     path: "{{ path }}" 
     recurse: "{{ file.recurse | default(true) }}" 
     mode: "{{ file.mode | default( ((file.state | default('directory')) == 'directory') | ternary('0755', '0644') ) }}"
-    owner: "{{ file.owner | default(kaas_project_config.file_owner) | default(kaas_default_file_owner) }}" 
-    group: "{{ file.group | default(kaas_project_config.file_group) | default(kaas_default_file_group) }}"
+    owner: "{{ owner }}" 
+    group: "{{ group }}"
     state: "{{ file.state | default('directory') }}"
diff --git a/roles/ands_kaas/templates/0-gfs-volumes.yml.j2 b/roles/ands_kaas/templates/0-gfs-volumes.yml.j2
index a162c8b..8e5842a 100644
--- a/roles/ands_kaas/templates/0-gfs-volumes.yml.j2
+++ b/roles/ands_kaas/templates/0-gfs-volumes.yml.j2
@@ -7,10 +7,11 @@ metadata:
     descriptions: "KATRIN Volumes"
 objects:
 {% for name, vol in (kaas_project_config.volumes | default(kaas_openshift_volumes)).iteritems() %}
+{% set oc_name = vol.name | default(name) | regex_replace('_','-') %}
   - apiVersion: v1
     kind: PersistentVolume
     metadata:
-      name: {{ vol.name | default(name) }}
+      name: {{ oc_name }}
     spec:
       persistentVolumeReclaimPolicy: Retain 
       glusterfs: 
@@ -22,14 +23,14 @@ objects:
       capacity:
         storage: {{ vol.capacity | default(kaas_default_volume_capacity) }}
       claimRef:
-        name: {{ vol.name | default(name) }}
+        name: {{ oc_name }}
         namespace: {{ kaas_project }}
   - apiVersion: v1
     kind: PersistentVolumeClaim
     metadata:
-      name: {{ vol.name | default(name) }}
+      name: {{ oc_name }}
     spec:
-      volumeName: {{ vol.name | default(name) }}
+      volumeName: {{ oc_name }}
       accessModes:
         - {{ vol.access | default('ReadWriteMany') }}
       resources:
diff --git a/roles/ands_kaas/templates/6-kaas-pods.yml.j2 b/roles/ands_kaas/templates/6-kaas-pods.yml.j2
index 479b343..d5418d3 100644
--- a/roles/ands_kaas/templates/6-kaas-pods.yml.j2
+++ b/roles/ands_kaas/templates/6-kaas-pods.yml.j2
@@ -36,7 +36,7 @@ objects:
   - apiVersion: v1
     kind: Route
     metadata:
-      name: kaas
+      name: {{ pod.name | default(name) }}
     spec:
       host: {{ pod.service.host }}
       to:
@@ -66,7 +66,7 @@ objects:
   - apiVersion: v1
     kind: DeploymentConfig
     metadata:
-      name: kaas
+      name: {{ pod.name | default(name) }}
     spec:
       replicas: {{ pod.sched.replicas | default(1) }}
       selector:
@@ -93,12 +93,33 @@ objects:
       {% for img in pod.images %}
         {% set imgidx = loop.index %}
         {% for vol in img.mappings %}
+            {% set oc_name = vol.name | default(name) | regex_replace('_','-') %}
             - name: vol-{{imgidx}}-{{loop.index}}
               persistentVolumeClaim: 
-                claimName: {{ vol.name }}
+                claimName: {{ oc_name }}
         {% endfor %}
       {% endfor %}
     {% endif %}
+    {% if (pod.groups is defined) or (pod.run_as is defined) %}
+          securityContext:
+        {% if (pod.run_as is defined) %}
+                {% if (kaas_project_config.uids | default(kaas_openshift_uids))[pod.run_as] is defined %}
+                - {{ (kaas_project_config.uids | default(kaas_openshift_uids))[pod.run_as].id }}
+                {% else %}
+                - pod.run_as
+                {% endif %}
+        {% endif %}
+        {% if (pod.groups is defined) %}
+            supplementalGroups:
+            {% for group in pod.groups %}
+                {% if (kaas_project_config.gids | default(kaas_openshift_gids))[group] is defined %}
+                - {{ (kaas_project_config.gids | default(kaas_openshift_gids))[group].id }}
+                {% else %}
+                - group
+                {% endif %}
+            {% endfor %}
+        {% endif %}
+    {% endif %}
           containers:
     {% for img in pod.images %}
       {% set imgidx = loop.index %}
@@ -118,10 +139,12 @@ objects:
       {% endif %}
       {% if img.env is defined %}
               env:
-        {% for env_name, env_val in img.env.iteritems() %}
+        {% for env_item in img.env %}
+          {% set env_name = env_item.name %}
+          {% set env_val = env_item.value %}
           {% set env_parts = (env_val | string).split('@') %}
+                - name: "{{ env_name }}"
           {% if env_parts[0] == "secret" %}
-                - name: {{ env_name }}
              {% set env_sec = (env_parts[1] | string).split('/') %}
                   valueFrom: 
                     secretKeyRef:
@@ -134,7 +157,7 @@ objects:
                       name: {{ env_cm[0] }}
                       key: {{ env_cm[1] }}
           {% else %}
-                  value: {{ env_val }}
+                  value: "{{ env_val }}"
           {% endif %}
         {% endfor %}
       {% endif %}
diff --git a/roles/ands_openshift/defaults/main.yml b/roles/ands_openshift/defaults/main.yml
index e473b98..b97b584 100644
--- a/roles/ands_openshift/defaults/main.yml
+++ b/roles/ands_openshift/defaults/main.yml
@@ -1,4 +1,4 @@
-openshift_common_subroles: "{{ [ 'hostnames', 'users', 'storage' ] }}"
+openshift_common_subroles: "{{ [ 'hostnames', 'users', 'security', 'storage' ] }}"
 openshift_heketi_subroles: "{{ [ 'ssh', 'heketi' ] }}"
 openshift_all_subroles: "{{ ands_configure_heketi | default(False) | ternary(openshift_common_subroles + openshift_heketi_subroles, openshift_common_subroles) }}"
 
@@ -9,5 +9,6 @@ openshift_namespace: "default"
 ssh_template_path: "{{ ands_paths.provision }}/ssh/"
 storage_template_path: "{{ ands_paths.provision }}/gfs/"
 heketi_template_path: "{{ ands_paths.provision }}/heketi/"
+ands_openshift_patch_path: "{{ ands_paths.provision }}/patch/"
 
 openshift_storage_nodes: "{{ groups.storage_nodes | map('extract', hostvars, 'ands_storage_hostname') | list }}"
diff --git a/roles/ands_openshift/tasks/security.yml b/roles/ands_openshift/tasks/security.yml
new file mode 100644
index 0000000..b1f017b
--- /dev/null
+++ b/roles/ands_openshift/tasks/security.yml
@@ -0,0 +1,3 @@
+- include_tasks: security_resources.yml
+  run_once: true
+  delegate_to: "{{ groups.masters[0] }}"
diff --git a/roles/ands_openshift/tasks/security_resources.yml b/roles/ands_openshift/tasks/security_resources.yml
new file mode 100644
index 0000000..5644723
--- /dev/null
+++ b/roles/ands_openshift/tasks/security_resources.yml
@@ -0,0 +1,54 @@
+---
+- name: Ensure OpenShift patch directory exists
+  file: path="{{ ands_openshift_patch_path }}" state="directory" mode=0644 owner=root group=root
+
+# No spaces in patch, otherwise escaping mess...
+- name: Patch group range in project configuration
+  include_role: name="openshift_resource" tasks_from="patch.yml" 
+  vars:
+    project: "{{ prj_item }}" 
+    resource: "ns/{{ prj_item }}"
+    patch: '{"metadata":{"annotations":{"openshift.io/sa.scc.supplemental-groups":"{{ands_openshift_gid_ranges[prj_item]}}"}}}'
+    patch_path: "{{ ands_openshift_patch_path }}"
+  with_items: "{{ (ands_openshift_gid_ranges | default({})).keys() }}"
+  loop_control: 
+    loop_var: prj_item
+
+- name: Patch uid range in project configuration
+  include_role: name="openshift_resource" tasks_from="patch.yml" 
+  vars:
+    project: "{{ prj_item }}" 
+    resource: "ns/{{ prj_item }}"
+    patch: '{"metadata":{"annotations":{"openshift.io/sa.scc.uid-range":"{{ands_openshift_uid_ranges[prj_item]}}"}}}'
+    patch_path: "{{ ands_openshift_patch_path }}"
+  with_items: "{{ (ands_openshift_uid_ranges | default({})).keys() }}"
+  loop_control: 
+    loop_var: prj_item
+
+- name: Restrict supplementalGroups
+  include_role: name="openshift_resource" tasks_from="patch.yml" 
+  vars:
+    project: "{{ prj_item }}" 
+    resource: "scc/restricted"
+    modes: "{{ ands_openshift_gid_mode | default({}) }}"
+    mode: "{{ (modes[prj_item] is defined) | ternary(modes[prj_item], modes['ands_default'] | default(false)) }}"
+    patch: '{"supplementalGroups":{"type":"{{mode}}"}}'
+    patch_path: "{{ ands_openshift_patch_path }}"
+  when: mode != false
+  with_items: "{{ (ands_openshift_projects | default({})).keys() }}"
+  loop_control: 
+    loop_var: prj_item
+
+- name: Configure runAsUser
+  include_role: name="openshift_resource" tasks_from="patch.yml" 
+  vars:
+    project: "{{ prj_item }}" 
+    resource: "scc/restricted"
+    modes: "{{ ands_openshift_uid_mode | default({}) }}"
+    mode: "{{ (modes[prj_item] is defined) | ternary(modes[prj_item], modes['ands_default'] | default(false)) }}"
+    patch: '{"runAsUser":{"type":"{{mode}}"}}'
+    patch_path: "{{ ands_openshift_patch_path }}"
+  when: mode != false
+  with_items: "{{ (ands_openshift_projects | default({})).keys() }}"
+  loop_control: 
+    loop_var: prj_item
diff --git a/roles/ands_openshift/tasks/users_resources.yml b/roles/ands_openshift/tasks/users_resources.yml
index 35323cb..5bc748c 100644
--- a/roles/ands_openshift/tasks/users_resources.yml
+++ b/roles/ands_openshift/tasks/users_resources.yml
@@ -2,7 +2,9 @@
 - name: Configure cluster roles
   command: "oc adm policy add-cluster-role-to-user {{  item.key.split('/')[0] }} {{ item.value.replace(' ','').split(',') | join(' ') }}"
   with_dict: "{{ ands_openshift_roles }}"
-  when: "{{ item.key.split('/') | length == 1 }}"
+  when: key_len == "1"
+  vars:
+    key_len: "{{ item.key.split('/') | length }}"
 
 - name: Get project list
   command: "oc get projects -o json"
@@ -20,7 +22,9 @@
 - name: Configure per project roles
   command: "oc adm policy add-role-to-user -n {{  item.key.split('/')[0] }} {{ item.key.split('/')[1] }} {{ item.value.replace(' ','').split(',') | join(' ') }}"
   with_dict: "{{ ands_openshift_roles }}"
-  when: "{{ item.key.split('/') | length == 2 }}"
+  when: key_len == "2"
+  vars:
+    key_len: "{{ item.key.split('/') | length }}"
 
 - name: Get user list
   command: "oc get users -o json"
@@ -31,10 +35,12 @@
   set_fact: removed_users="{{ results.stdout | from_json | json_query('items[*].metadata.name') | difference(ands_openshift_users.keys()) }}"
   when: (results | succeeded)
 
-- name: Create missing projects
+- name: Remove user authentication
   command: "oc delete identity htpasswd_auth:{{ item }}"
   with_items: "{{ removed_users | default([]) }}"
 
-- name: Create missing projects
+- name: Remove users
   command: "oc delete user {{ item }}"
   with_items: "{{ removed_users | default([]) }}"
+
+
diff --git a/roles/glusterfs/tasks/data b/roles/glusterfs/tasks/data
new file mode 120000
index 0000000..31bb52e
--- /dev/null
+++ b/roles/glusterfs/tasks/data
@@ -0,0 +1 @@
+cfg
\ No newline at end of file
diff --git a/roles/glusterfs/tasks/db/vols3.yml b/roles/glusterfs/tasks/db/vols3.yml
new file mode 100644
index 0000000..b1beacb
--- /dev/null
+++ b/roles/glusterfs/tasks/db/vols3.yml
@@ -0,0 +1,14 @@
+---
+- name: "Create {{ name }} volume"
+  gluster_volume: 
+    state: present
+    name: "{{ name }}"
+    cluster: "{{ domain_servers | join(',') }}"
+    disperses: "3"
+    redundancies: "1"
+    bricks: "{{ glusterfs_bricks_path }}/brick-{{ name }}"
+    transport: "{{ glusterfs_transport }}"
+
+
+- name: "Start {{ name }} volume"
+  gluster_volume: state="started" name="{{ name }}"
diff --git a/roles/openshift_resource/defaults/main.yml b/roles/openshift_resource/defaults/main.yml
index ec44c4f..7994827 100644
--- a/roles/openshift_resource/defaults/main.yml
+++ b/roles/openshift_resource/defaults/main.yml
@@ -1 +1,2 @@
-template_path: "/mnt/provision/templates"
+template_path: "{{ ands_paths.provision }}/templates"
+patch_path: "{{ ands_paths.provision }}/patches"
\ No newline at end of file
diff --git a/roles/openshift_resource/tasks/patch.yml b/roles/openshift_resource/tasks/patch.yml
new file mode 100644
index 0000000..e2bbcfa
--- /dev/null
+++ b/roles/openshift_resource/tasks/patch.yml
@@ -0,0 +1,41 @@
+---
+- name: Lookup the specified resource
+  command: "oc get -n '{{project}}' '{{resource}}' -o json"
+  register: orig_result
+  changed_when: 0
+
+- name: Lookup API version of the specified resource
+  command: "oc get -n '{{project}}' '{{resource}}' --template {{'{{' + '.apiVersion' + '}}'}}"
+  register: api_version
+  changed_when: 0
+
+# Fucking ansible is making mess of escaping. Main problem it parses to objects strings starting with '{ ... }', but not with ' { ... }'
+- name: Escaping patch
+  set_fact: xpatch='{{patch | to_json | regex_replace(" ","") | regex_replace("^", " ")}}'
+
+- name: Generate dummy patch {{resource}} in {{project}}
+  command: "oc patch -n '{{project}}' --patch ' {\"apiVersion\": \"{{api_version.stdout}}\"}' --local=true -f - -o json"
+  args:
+    stdin: " {{ orig_result.stdout_lines | join('')  }}"
+  register: dummy_result
+  changed_when: 0
+
+- name: Generate test patch {{resource}} in {{project}}
+  command: "oc patch -n '{{project}}' --patch '{{xpatch}}' --local=true -f - -o json"
+  args:
+    stdin: " {{ orig_result.stdout_lines | join('')  }}"
+  register: patch_result
+  changed_when: 0
+
+#- debug: msg="{{ dummy_result.stdout }}"
+#  when: dummy_result.stdout != patch_result.stdout
+  
+#- debug: msg="{{ patch_result.stdout }}"
+#  when: dummy_result.stdout != patch_result.stdout
+
+- name: Patch {{resource}} in {{project}}
+  command: "oc patch -n '{{project}}' '{{resource}}' --patch '{{xpatch}}'"
+  register: result
+  changed_when: (result | succeeded)
+  when: dummy_result.stdout != patch_result.stdout
+  
\ No newline at end of file
diff --git a/setup.sh b/setup.sh
index 2905cda..20174f1 100755
--- a/setup.sh
+++ b/setup.sh
@@ -47,6 +47,9 @@ case "${1}" in
     users)
         apply playbooks/openshift-setup-users.yml || exit 1
         ;;
+    security)
+        apply playbooks/openshift-setup-security.yml || exit 1
+        ;;
     storage)
         apply playbooks/openshift-setup-storage.yml || exit 1
         ;;
diff --git a/setup/configs/openshift.yml b/setup/configs/openshift.yml
index 6b9995c..e2a2d6d 100644
--- a/setup/configs/openshift.yml
+++ b/setup/configs/openshift.yml
@@ -2,7 +2,8 @@
 ands_openshift_projects:
     katrin: KArlsruhe TRItium Neutrino
     adei: ADEI
-    
+
+#    test: Tesing    
 
 ands_openshift_users:
   pdv: IPE Administation Account
diff --git a/setup/configs/security.yml b/setup/configs/security.yml
new file mode 100644
index 0000000..413f57e
--- /dev/null
+++ b/setup/configs/security.yml
@@ -0,0 +1,21 @@
+ands_openshift_gid_mode:
+#    adei: "RunAsAny"
+    ands_default: "MustRunAs"
+
+#ands_openshift_uid_mode:
+#    ands_default: "MustRunAsRange"
+
+#ands_openshift_uid_ranges:
+
+ands_openshift_gid_ranges:
+  katrin: "5000/10"
+  adei: "5010/10"
+
+ands_openshift_uids:
+  kaas: { id: 6000 }
+
+ands_openshift_gids:
+  kaas: { id: 6000 }
+
+ands_default_file_group: root
+ands_default_file_owner: root
diff --git a/setup/configs/volumes.yml b/setup/configs/volumes.yml
index d0ba063..d93f177 100644
--- a/setup/configs/volumes.yml
+++ b/setup/configs/volumes.yml
@@ -3,6 +3,8 @@ ands_paths:
   provision: /mnt/provision
   openshift: /mnt/openshift
   temporary: /mnt/temporary
+  databases: /mnt/databases
+  katrin_data: /mnt/katrin
 
 ands_heketi_domain:
   servers: "storage_nodes"
@@ -14,11 +16,15 @@ ands_storage_domains:
     clients: "masters"
     volumes:
       provision: { type: "cfg", mount: "{{ ands_paths.provision }}" }
-  - servers: "storage_nodes"
-    clients: "nodes"
-    volumes: 
       openshift: { type: "cfg", mount: "{{ ands_paths.openshift }}" }
+      databases: { type: "db",  mount: "{{ ands_paths.databases }}" }
       temporary: { type: "tmp", mount: "{{ ands_paths.temporary }}" }
+      katrin_data: { type: "data", mount: "{{ ands_paths.katrin_data }}" }
+#  - servers: "storage_nodes"
+#    clients: "nodes"
+#      openshift: { type: "cfg", mount: "{{ ands_paths.openshift }}" }
+#      temporary: { type: "tmp", mount: "{{ ands_paths.temporary }}" }
+#    volumes: 
 #  - ovirt:
 #  - pdv:
 
@@ -31,7 +37,6 @@ ands_openshift_volumes:
   log: { volume: "temporary", path: "/log", write: true}
   tmp: { volume: "temporary", path: "/tmp", write: true}
 
-
 # Global list, we only take things from the volume of project
 ands_openshift_files:
   - { osv: "log", path: "apache2-kaas", state: "directory", mode: "0777" }
diff --git a/setup/projects/adei/vars/pods.yml b/setup/projects/adei/vars/pods.yml
new file mode 100644
index 0000000..3b104ea
--- /dev/null
+++ b/setup/projects/adei/vars/pods.yml
@@ -0,0 +1,52 @@
+volumes:
+  adei_etc: { volume: "openshift", path: "/adei/etc" }
+  adei_db:  { volume: "databases", path: "/adei", write: true}
+
+gids:
+  adei: { id: 5010 }
+  adei_db: { id: 5011 }
+
+files:
+  - { osv: "adei_db", path: "mysql", state: "directory", group: "adei_db", mode: "0775" }
+
+
+pods:
+  mysql:
+    service: { ports: [ 3306 ] }
+    sched: { replicas: 1, selector: { master: 1 } }
+    selector: { master: 1 }
+    groups: [ "adei_db" ]
+    images:
+      - image: "openshift/mysql-56-centos7"
+        env:
+            - { name: "MYSQL_USER", value: "adei" }
+            - { name: "MYSQL_PASSWORD", value: "adei" } 
+            - { name: "MYSQL_DATABASE", value: "adei" }
+        mappings: 
+            - { name: "adei_etc", path: "mysql", mount: "/etc/mysql" }
+            - { name: "adei_db", path: "mysql", mount: "/var/lib/mysql/data" }
+        probes:
+            - { port: 3306 }
+  phpmyadmin:
+    service: { host: "phpmyadmin.{{ openshift_master_default_subdomain }}", ports: [ 80/8080 ] }
+    sched: { replicas: 1 }
+    images:
+      - image: "chsa/phpmyadmin-centos:4"
+        env:
+            - { name: "DB_SERVICE_HOST", value: "mysql.adei.svc.cluster.local" }
+            - { name: "DB_SERVICE_PORT", value: "3306" } 
+            - { name: "DB_SERVICE_CONTROL_USER", value: "pma" }
+            - { name: "DB_SERVICE_CONTROL_PASSWORD", value: "adei" }
+            - { name: "DB_EXTRA_HOSTS", value: "mysql.katrin.svc.cluster.local" }
+        probes:
+            - { port: 8080, path: '/' }
+
+
+
+#oc:
+#  - template: "[0-3]*"
+#  - template: "[4-6]*"
+#  - resource: "route/apache" 
+#    oc: "expose svc/kaas --name apache --hostname=apache.{{ openshift_master_default_subdomain }}"
+#  - template: "*"
+  
\ No newline at end of file