Install fluentd on nodes and master

* Configure fluentd to aggragate container logs

8 files changed, 158 insertions, 0 deletions

diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 05822d118..2566f979e 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -6,6 +6,7 @@
   roles:
   - openshift_master
   - { role: openshift_sdn_master, when: openshift.common.use_openshift_sdn | bool }
+  - fluentd_master
   tasks:
   - name: Create group for deployment type
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
diff --git a/playbooks/common/openshift-node/config.yml b/playbooks/common/openshift-node/config.yml
index 96641a274..b371312bc 100644
--- a/playbooks/common/openshift-node/config.yml
+++ b/playbooks/common/openshift-node/config.yml
@@ -96,6 +96,7 @@
   roles:
   - openshift_node
   - { role: openshift_sdn_node, when: openshift.common.use_openshift_sdn | bool }
+  - fluentd_node
   tasks:
   - name: Create group for deployment type
     group_by: key=oo_nodes_deployment_type_{{ openshift.common.deployment_type }}
diff --git a/roles/fluentd_master/tasks/main.yml b/roles/fluentd_master/tasks/main.yml
new file mode 100644
index 000000000..9c558d92d
--- /dev/null
+++ b/roles/fluentd_master/tasks/main.yml
@@ -0,0 +1,40 @@
+---
+# TODO: Update fluentd install and configuration when packaging is complete
+- name: download and install td-agent
+  yum:
+    name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm'
+    state: present
+
+- name: install Kubernetes fluentd plugin
+  command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes'
+
+- name: Creates directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0755
+  with_items: ['/etc/td-agent/config.d']
+
+- name: Add include to td-agent configuration
+  lineinfile:
+    dest: '/etc/td-agent/td-agent.conf'
+    regexp: '^@include config.d'
+    line: '@include config.d/*.conf'
+    state: present
+
+- name: install Kubernetes fluentd configuration file
+  template:
+    src: kubernetes.conf.j2
+    dest: /etc/td-agent/config.d/kubernetes.conf
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: ensure td-agent is running
+  service:
+    name: 'td-agent'
+    state: started
+    enabled: yes
+
diff --git a/roles/fluentd_master/templates/kubernetes.conf.j2 b/roles/fluentd_master/templates/kubernetes.conf.j2
new file mode 100644
index 000000000..7b5c86062
--- /dev/null
+++ b/roles/fluentd_master/templates/kubernetes.conf.j2
@@ -0,0 +1,9 @@
+<match kubernetes.**>
+    type file
+    path /var/log/td-agent/containers.log
+    time_slice_format %Y%m%d
+    time_slice_wait 10m
+    time_format %Y%m%dT%H%M%S%z
+    compress gzip
+    utc
+</match>
diff --git a/roles/fluentd_node/tasks/main.yml b/roles/fluentd_node/tasks/main.yml
new file mode 100644
index 000000000..951c5a229
--- /dev/null
+++ b/roles/fluentd_node/tasks/main.yml
@@ -0,0 +1,48 @@
+---
+# TODO: Update fluentd install and configuration when packaging is complete
+- name: download and install td-agent
+  yum:
+    name: 'http://packages.treasuredata.com/2/redhat/7/x86_64/td-agent-2.2.0-0.x86_64.rpm'
+    state: present
+
+- name: install Kubernetes fluentd plugin
+  command: '/opt/td-agent/embedded/bin/gem install fluent-plugin-kubernetes'
+
+- name: Override td-agent configuration file
+  template:
+    src: td-agent.j2
+    dest: /etc/sysconfig/td-agent
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: Creates directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0755
+  with_items: ['/etc/td-agent/config.d', '/var/log/td-agent/tmp']
+
+- name: Add include to td-agent configuration
+  lineinfile:
+    dest: '/etc/td-agent/td-agent.conf'
+    regexp: '^@include config.d'
+    line: '@include config.d/*.conf'
+    state: present
+
+- name: install Kubernetes fluentd configuration file
+  template:
+    src: kubernetes.conf.j2
+    dest: /etc/td-agent/config.d/kubernetes.conf
+    group: 'td-agent'
+    owner: 'td-agent'
+    mode: 0444
+
+- name: ensure td-agent is running
+  service:
+    name: 'td-agent'
+    state: started
+    enabled: yes
+
diff --git a/roles/fluentd_node/templates/kubernetes.conf.j2 b/roles/fluentd_node/templates/kubernetes.conf.j2
new file mode 100644
index 000000000..5f1eecb20
--- /dev/null
+++ b/roles/fluentd_node/templates/kubernetes.conf.j2
@@ -0,0 +1,53 @@
+<source>
+  type tail
+  path /var/lib/docker/containers/*/*-json.log
+  pos_file /var/log/td-agent/tmp/fluentd-docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag docker.*
+  format json
+  read_from_head true
+</source>
+
+<match docker.var.lib.docker.containers.*.*.log>
+  type kubernetes
+  container_id ${tag_parts[5]}
+  tag docker.${name}
+</match>
+
+<match kubernetes>
+  type copy
+
+  <store>
+    type forward
+    send_timeout 60s
+    recover_wait 10s
+    heartbeat_interval 1s
+    phi_threshold 16
+    hard_timeout 60s
+    log_level trace
+    require_ack_response true
+    heartbeat_type tcp
+
+    <server>
+      name {{groups['oo_first_master'][0]}}
+      host {{hostvars[groups['oo_first_master'][0]].openshift.common.hostname}}
+      port 24224
+      weight 60
+    </server>
+
+    <secondary>
+      type file
+      path /var/log/td-agent/forward-failed
+    </secondary>
+  </store>
+
+  <store>
+    type file
+    path /var/log/td-agent/containers.log
+    time_slice_format %Y%m%d
+    time_slice_wait 10m
+    time_format %Y%m%dT%H%M%S%z
+    compress gzip
+    utc
+  </store>
+</match>
diff --git a/roles/fluentd_node/templates/td-agent.j2 b/roles/fluentd_node/templates/td-agent.j2
new file mode 100644
index 000000000..7245e11ec
--- /dev/null
+++ b/roles/fluentd_node/templates/td-agent.j2
@@ -0,0 +1,2 @@
+DAEMON_ARGS=
+TD_AGENT_ARGS="/usr/sbin/td-agent --log /var/log/td-agent/td-agent.log --use-v1-config"
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 56cf43531..11195e83e 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -11,6 +11,10 @@ os_firewall_allow:
   port: 53/tcp
 - service: OpenShift dns udp
   port: 53/udp
+- service: Fluentd td-agent tcp
+  port: 24224/tcp
+- service: Fluentd td-agent udp
+  port: 24224/udp
 os_firewall_deny:
 - service: OpenShift api http
   port: 8080/tcp