summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorScott Dodson <sdodson@redhat.com>2017-07-28 21:01:11 -0400
committerGitHub <noreply@github.com>2017-07-28 21:01:11 -0400
commit1990a975c3101389631eaee925760607600a0629 (patch)
tree71229bbd384693a8ab041bcca7183c5d8a7d5cd3
parentb5850e6cae2998e83702f2562ab32d1a1687a572 (diff)
parent61fe309c853946585a2a6a6404ef89dc5ebfb9f0 (diff)
downloadopenshift-1990a975c3101389631eaee925760607600a0629.tar.gz
openshift-1990a975c3101389631eaee925760607600a0629.tar.bz2
openshift-1990a975c3101389631eaee925760607600a0629.tar.xz
openshift-1990a975c3101389631eaee925760607600a0629.zip
Merge pull request #4943 from jcantrill/file_buffer_master
Impl fluentd file buffer
-rw-r--r--roles/openshift_logging/README.md16
-rw-r--r--roles/openshift_logging_fluentd/defaults/main.yml2
-rw-r--r--roles/openshift_logging_fluentd/templates/fluentd.j27
-rw-r--r--roles/openshift_logging_mux/defaults/main.yml17
-rw-r--r--roles/openshift_logging_mux/tasks/main.yaml12
-rw-r--r--roles/openshift_logging_mux/templates/mux.j214
6 files changed, 68 insertions, 0 deletions
diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md
index d2ef7cc71..97650e2ce 100644
--- a/roles/openshift_logging/README.md
+++ b/roles/openshift_logging/README.md
@@ -57,6 +57,7 @@ When both `openshift_logging_install_logging` and `openshift_logging_upgrade_log
- `openshift_logging_fluentd_hosts`: List of nodes that should be labeled for Fluentd to be deployed to. Defaults to ['--all'].
- `openshift_logging_fluentd_buffer_queue_limit`: Buffer queue limit for Fluentd. Defaults to 1024.
- `openshift_logging_fluentd_buffer_size_limit`: Buffer chunk limit for Fluentd. Defaults to 1m.
+- `openshift_logging_fluentd_file_buffer_limit`: Fluentd will set the value to the file buffer limit. Defaults to '1Gi' per destination.
- `openshift_logging_es_host`: The name of the ES service Fluentd should send logs to. Defaults to 'logging-es'.
@@ -160,3 +161,18 @@ Elasticsearch OPS too, if using an OPS cluster:
need to set this
- `openshift_logging_mux_buffer_queue_limit`: Default `[1024]` - Buffer queue limit for Mux.
- `openshift_logging_mux_buffer_size_limit`: Default `[1m]` - Buffer chunk limit for Mux.
+- `openshift_logging_mux_file_buffer_limit`: Default `[2Gi]` per destination - Mux will
+ set the value to the file buffer limit.
+- `openshift_logging_mux_file_buffer_storage_type`: Default `[emptydir]` - Storage
+ type for the file buffer. One of [`emptydir`, `pvc`, `hostmount`]
+
+- `openshift_logging_mux_file_buffer_pvc_size`: The requested size for the file buffer
+ PVC, when not provided the role will not generate any PVCs. Defaults to `4Gi`.
+- `openshift_logging_mux_file_buffer_pvc_dynamic`: Whether or not to add the dynamic
+ PVC annotation for any generated PVCs. Defaults to 'False'.
+- `openshift_logging_mux_file_buffer_pvc_pv_selector`: A key/value map added to a PVC
+ in order to select specific PVs. Defaults to 'None'.
+- `openshift_logging_mux_file_buffer_pvc_prefix`: The prefix for the generated PVCs.
+ Defaults to 'logging-mux'.
+- `openshift_logging_mux_file_buffer_storage_group`: The storage group used for Mux.
+ Defaults to '65534'.
diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml
index ce7cfc433..be9943b0d 100644
--- a/roles/openshift_logging_fluentd/defaults/main.yml
+++ b/roles/openshift_logging_fluentd/defaults/main.yml
@@ -57,3 +57,5 @@ openshift_logging_fluentd_es_copy: false
#fluentd_config_contents:
#fluentd_throttle_contents:
#fluentd_secureforward_contents:
+
+openshift_logging_fluentd_file_buffer_limit: 1Gi
diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2
index 970e5c2a5..a4cf9a149 100644
--- a/roles/openshift_logging_fluentd/templates/fluentd.j2
+++ b/roles/openshift_logging_fluentd/templates/fluentd.j2
@@ -62,6 +62,8 @@ spec:
- name: dockerdaemoncfg
mountPath: /etc/docker
readOnly: true
+ - name: filebufferstorage
+ mountPath: /var/lib/fluentd
{% if openshift_logging_use_mux_client | bool %}
- name: muxcerts
mountPath: /etc/fluent/muxkeys
@@ -112,6 +114,8 @@ spec:
resource: limits.memory
- name: "USE_MUX_CLIENT"
value: "{{ openshift_logging_use_mux_client | default('false') | lower }}"
+ - name: "FILE_BUFFER_LIMIT"
+ value: "{{ openshift_logging_fluentd_file_buffer_limit | default('1Gi') }}"
volumes:
- name: runlogjournal
hostPath:
@@ -145,3 +149,6 @@ spec:
secret:
secretName: logging-mux
{% endif %}
+ - name: filebufferstorage
+ hostPath:
+ path: "/var/lib/fluentd"
diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml
index 797a27c1b..35fc7146f 100644
--- a/roles/openshift_logging_mux/defaults/main.yml
+++ b/roles/openshift_logging_mux/defaults/main.yml
@@ -47,3 +47,20 @@ openshift_logging_mux_ops_ca: /etc/fluent/keys/ca
#mux_config_contents:
#mux_throttle_contents:
#mux_secureforward_contents:
+
+# One of ['emptydir', 'pvc', 'hostmount']
+openshift_logging_mux_file_buffer_storage_type: "emptydir"
+
+# pvc options
+# the name of the PVC we will bind to -- create it if it does not exist
+openshift_logging_mux_file_buffer_pvc_name: "logging-mux-pvc"
+
+# required if the PVC does not already exist
+openshift_logging_mux_file_buffer_pvc_size: 4Gi
+openshift_logging_mux_file_buffer_pvc_dynamic: false
+openshift_logging_mux_file_buffer_pvc_pv_selector: {}
+openshift_logging_mux_file_buffer_pvc_access_modes: ['ReadWriteOnce']
+openshift_logging_mux_file_buffer_storage_group: '65534'
+
+openshift_logging_mux_file_buffer_pvc_prefix: "logging-mux"
+openshift_logging_mux_file_buffer_limit: 2Gi
diff --git a/roles/openshift_logging_mux/tasks/main.yaml b/roles/openshift_logging_mux/tasks/main.yaml
index 02815806a..8ec93de7d 100644
--- a/roles/openshift_logging_mux/tasks/main.yaml
+++ b/roles/openshift_logging_mux/tasks/main.yaml
@@ -177,6 +177,18 @@
check_mode: no
changed_when: no
+- name: Create Mux PVC
+ oc_pvc:
+ state: present
+ name: "{{ openshift_logging_mux_file_buffer_pvc_name }}"
+ namespace: "{{ openshift_logging_mux_namespace }}"
+ volume_capacity: "{{ openshift_logging_mux_file_buffer_pvc_size }}"
+ access_modes: "{{ openshift_logging_mux_file_buffer_pvc_access_modes | list }}"
+ selector: "{{ openshift_logging_mux_file_buffer_pvc_pv_selector }}"
+ storage_class_name: "{{ openshift_logging_mux_file_buffer_pvc_storage_class_name | default('', true) }}"
+ when:
+ - openshift_logging_mux_file_buffer_storage_type == "pvc"
+
- name: Set logging-mux DC
oc_obj:
state: present
diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2
index 2b3b64bb8..e43d9d397 100644
--- a/roles/openshift_logging_mux/templates/mux.j2
+++ b/roles/openshift_logging_mux/templates/mux.j2
@@ -66,6 +66,8 @@ spec:
- name: muxcerts
mountPath: /etc/fluent/muxkeys
readOnly: true
+ - name: filebufferstorage
+ mountPath: /var/lib/fluentd
env:
- name: "K8S_HOST_URL"
value: "{{openshift_logging_mux_master_url}}"
@@ -115,6 +117,8 @@ spec:
resourceFieldRef:
containerName: "mux"
resource: limits.memory
+ - name: "FILE_BUFFER_LIMIT"
+ value: "{{ openshift_logging_mux_file_buffer_limit | default('2Gi') }}"
volumes:
- name: config
configMap:
@@ -131,3 +135,13 @@ spec:
- name: muxcerts
secret:
secretName: logging-mux
+ - name: filebufferstorage
+{% if openshift_logging_mux_file_buffer_storage_type == 'pvc' %}
+ persistentVolumeClaim:
+ claimName: {{ openshift_logging_mux_file_buffer_pvc_name }}
+{% elif openshift_logging_mux_file_buffer_storage_type == 'hostmount' %}
+ hostPath:
+ path: "/var/log/fluentd"
+{% else %}
+ emptydir: {}
+{% endif %}