Merge pull request #4943 from jcantrill/file_buffer_master

Impl fluentd file buffer

6 files changed, 68 insertions, 0 deletions

diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md
index d2ef7cc71..97650e2ce 100644
--- a/roles/openshift_logging/README.md
+++ b/roles/openshift_logging/README.md
@@ -57,6 +57,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log
 - `openshift_logging_fluentd_hosts`: List of nodes that should be labeled for Fluentd to be deployed to. Defaults to ['--all'].
 - `openshift_logging_fluentd_buffer_queue_limit`: Buffer queue limit for Fluentd. Defaults to 1024.
 - `openshift_logging_fluentd_buffer_size_limit`: Buffer chunk limit for Fluentd. Defaults to 1m.
+- `openshift_logging_fluentd_file_buffer_limit`: Fluentd will set the value to the file buffer limit.  Defaults to '1Gi' per destination.
 
 
 - `openshift_logging_es_host`: The name of the ES service Fluentd should send logs to. Defaults to 'logging-es'.
@@ -160,3 +161,18 @@ Elasticsearch OPS too, if using an OPS cluster:
   need to set this
 - `openshift_logging_mux_buffer_queue_limit`: Default `[1024]` - Buffer queue limit for Mux.
 - `openshift_logging_mux_buffer_size_limit`: Default `[1m]` - Buffer chunk limit for Mux.
+- `openshift_logging_mux_file_buffer_limit`: Default `[2Gi]` per destination - Mux will
+  set the value to the file buffer limit.
+- `openshift_logging_mux_file_buffer_storage_type`: Default `[emptydir]` - Storage
+  type for the file buffer.  One of [`emptydir`, `pvc`, `hostmount`]
+
+- `openshift_logging_mux_file_buffer_pvc_size`: The requested size for the file buffer
+  PVC, when not provided the role will not generate any PVCs. Defaults to `4Gi`.
+- `openshift_logging_mux_file_buffer_pvc_dynamic`: Whether or not to add the dynamic
+  PVC annotation for any generated PVCs. Defaults to 'False'.
+- `openshift_logging_mux_file_buffer_pvc_pv_selector`: A key/value map added to a PVC
+  in order to select specific PVs.  Defaults to 'None'.
+- `openshift_logging_mux_file_buffer_pvc_prefix`: The prefix for the generated PVCs.
+  Defaults to 'logging-mux'.
+- `openshift_logging_mux_file_buffer_storage_group`: The storage group used for Mux.
+  Defaults to '65534'.
diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml
index ce7cfc433..be9943b0d 100644
--- a/roles/openshift_logging_fluentd/defaults/main.yml
+++ b/roles/openshift_logging_fluentd/defaults/main.yml
@@ -57,3 +57,5 @@ openshift_logging_fluentd_es_copy: false
 #fluentd_config_contents:
 #fluentd_throttle_contents:
 #fluentd_secureforward_contents:
+
+openshift_logging_fluentd_file_buffer_limit: 1Gi
diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2
index 970e5c2a5..a4cf9a149 100644
--- a/roles/openshift_logging_fluentd/templates/fluentd.j2
+++ b/roles/openshift_logging_fluentd/templates/fluentd.j2
@@ -62,6 +62,8 @@ spec:
         - name: dockerdaemoncfg
           mountPath: /etc/docker
           readOnly: true
+        - name: filebufferstorage
+          mountPath: /var/lib/fluentd
 {% if openshift_logging_use_mux_client | bool %}
         - name: muxcerts
           mountPath: /etc/fluent/muxkeys
@@ -112,6 +114,8 @@ spec:
               resource: limits.memory
         - name: "USE_MUX_CLIENT"
           value: "{{ openshift_logging_use_mux_client | default('false') | lower }}"
+        - name: "FILE_BUFFER_LIMIT"
+          value: "{{ openshift_logging_fluentd_file_buffer_limit | default('1Gi') }}"
       volumes:
       - name: runlogjournal
         hostPath:
@@ -145,3 +149,6 @@ spec:
         secret:
           secretName: logging-mux
 {% endif %}
+      - name: filebufferstorage
+        hostPath:
+          path: "/var/lib/fluentd"
diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml
index 797a27c1b..35fc7146f 100644
--- a/roles/openshift_logging_mux/defaults/main.yml
+++ b/roles/openshift_logging_mux/defaults/main.yml
@@ -47,3 +47,20 @@ openshift_logging_mux_ops_ca: /etc/fluent/keys/ca
 #mux_config_contents:
 #mux_throttle_contents:
 #mux_secureforward_contents:
+
+# One of ['emptydir', 'pvc', 'hostmount']
+openshift_logging_mux_file_buffer_storage_type: "emptydir"
+
+# pvc options
+# the name of the PVC we will bind to -- create it if it does not exist
+openshift_logging_mux_file_buffer_pvc_name: "logging-mux-pvc"
+
+# required if the PVC does not already exist
+openshift_logging_mux_file_buffer_pvc_size: 4Gi
+openshift_logging_mux_file_buffer_pvc_dynamic: false
+openshift_logging_mux_file_buffer_pvc_pv_selector: {}
+openshift_logging_mux_file_buffer_pvc_access_modes: ['ReadWriteOnce']
+openshift_logging_mux_file_buffer_storage_group: '65534'
+
+openshift_logging_mux_file_buffer_pvc_prefix: "logging-mux"
+openshift_logging_mux_file_buffer_limit: 2Gi
diff --git a/roles/openshift_logging_mux/tasks/main.yaml b/roles/openshift_logging_mux/tasks/main.yaml
index 02815806a..8ec93de7d 100644
--- a/roles/openshift_logging_mux/tasks/main.yaml
+++ b/roles/openshift_logging_mux/tasks/main.yaml
@@ -177,6 +177,18 @@
   check_mode: no
   changed_when: no
 
+- name: Create Mux PVC
+  oc_pvc:
+    state: present
+    name: "{{ openshift_logging_mux_file_buffer_pvc_name }}"
+    namespace: "{{ openshift_logging_mux_namespace }}"
+    volume_capacity: "{{ openshift_logging_mux_file_buffer_pvc_size }}"
+    access_modes: "{{ openshift_logging_mux_file_buffer_pvc_access_modes | list }}"
+    selector: "{{ openshift_logging_mux_file_buffer_pvc_pv_selector }}"
+    storage_class_name: "{{ openshift_logging_mux_file_buffer_pvc_storage_class_name | default('', true) }}"
+  when:
+  - openshift_logging_mux_file_buffer_storage_type == "pvc"
+
 - name: Set logging-mux DC
   oc_obj:
     state: present
diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2
index 2b3b64bb8..e43d9d397 100644
--- a/roles/openshift_logging_mux/templates/mux.j2
+++ b/roles/openshift_logging_mux/templates/mux.j2
@@ -66,6 +66,8 @@ spec:
         - name: muxcerts
           mountPath: /etc/fluent/muxkeys
           readOnly: true
+        - name: filebufferstorage
+          mountPath: /var/lib/fluentd
         env:
         - name: "K8S_HOST_URL"
           value: "{{openshift_logging_mux_master_url}}"
@@ -115,6 +117,8 @@ spec:
             resourceFieldRef:
               containerName: "mux"
               resource: limits.memory
+        - name: "FILE_BUFFER_LIMIT"
+          value: "{{ openshift_logging_mux_file_buffer_limit | default('2Gi') }}"
       volumes:
       - name: config
         configMap:
@@ -131,3 +135,13 @@ spec:
       - name: muxcerts
         secret:
           secretName: logging-mux
+      - name: filebufferstorage
+{% if openshift_logging_mux_file_buffer_storage_type == 'pvc' %}
+        persistentVolumeClaim:
+          claimName: {{ openshift_logging_mux_file_buffer_pvc_name }}
+{% elif openshift_logging_mux_file_buffer_storage_type == 'hostmount' %}
+        hostPath:
+          path: "/var/log/fluentd"
+{% else %}
+        emptydir: {}
+{% endif %}