8 files changed, 27 insertions, 13 deletions

diff --git a/roles/openshift_logging/README.md b/roles/openshift_logging/README.md
index 97650e2ce..84ead3548 100644
--- a/roles/openshift_logging/README.md
+++ b/roles/openshift_logging/README.md
@@ -135,16 +135,23 @@ Elasticsearch OPS too, if using an OPS cluster:
   secure_forward forwarder for the node agent Fluentd daemonsets running in the
   cluster.  This can be used to reduce the number of connections to the
   OpenShift API server, by using `mux` and configuring each node Fluentd to
-  send raw logs to mux and turn off the k8s metadata plugin.
+  send raw logs to mux and turn off the k8s metadata plugin.  This requires the
+  use of `openshift_logging_mux_client_mode` (see below).
 - `openshift_logging_mux_allow_external`: Default `False`.  If this is `True`,
   the `mux` service will be deployed, and it will be configured to allow
   Fluentd clients running outside of the cluster to send logs using
   secure_forward.  This allows OpenShift logging to be used as a central
   logging service for clients other than OpenShift, or other OpenShift
   clusters.
-- `openshift_logging_use_mux_client`: Default `False`.  If this is `True`, the
-  node agent Fluentd services will be configured to send logs to the mux
-  service rather than directly to Elasticsearch.
+- `openshift_logging_mux_client_mode`: Values - `minimal`, `maximal`.
+  Default is unset.  Setting this value will cause the Fluentd node agent to
+  send logs to mux rather than directly to Elasticsearch.  The value
+  `maximal` means that Fluentd will do as much processing as possible at the
+  node before sending the records to mux.  This is the current recommended
+  way to use mux due to current scaling issues.
+  The value `minimal` means that Fluentd will do *no* processing at all, and
+  send the raw logs to mux for processing.  We do not currently recommend using
+  this mode, and ansible will warn you about this.
 - `openshift_logging_mux_hostname`: Default is "mux." +
   `openshift_master_default_subdomain`.  This is the hostname *external*_
   clients will use to connect to mux, and will be used in the TLS server cert
diff --git a/roles/openshift_logging/defaults/main.yml b/roles/openshift_logging/defaults/main.yml
index 1c243f934..8b0f4cb62 100644
--- a/roles/openshift_logging/defaults/main.yml
+++ b/roles/openshift_logging/defaults/main.yml
@@ -157,8 +157,6 @@ openshift_logging_storage_access_modes: "{{ openshift_hosted_logging_storage_acc
 # mux - secure_forward listener service
 openshift_logging_mux_allow_external: False
 openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}"
-# this tells the fluentd node agent to use mux instead of sending directly to Elasticsearch
-openshift_logging_use_mux_client: False
 openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}"
 openshift_logging_mux_port: 24284
 openshift_logging_mux_cpu_limit: 500m
diff --git a/roles/openshift_logging_fluentd/defaults/main.yml b/roles/openshift_logging_fluentd/defaults/main.yml
index be9943b0d..a53bbd2df 100644
--- a/roles/openshift_logging_fluentd/defaults/main.yml
+++ b/roles/openshift_logging_fluentd/defaults/main.yml
@@ -48,7 +48,6 @@ openshift_logging_fluentd_aggregating_strict: "no"
 openshift_logging_fluentd_aggregating_cert_path: none
 openshift_logging_fluentd_aggregating_key_path: none
 openshift_logging_fluentd_aggregating_passphrase: none
-openshift_logging_use_mux_client: False
 
 ### Deprecating in 3.6
 openshift_logging_fluentd_es_copy: false
diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml
index 55de2ae8d..9dfc6fc86 100644
--- a/roles/openshift_logging_fluentd/tasks/main.yaml
+++ b/roles/openshift_logging_fluentd/tasks/main.yaml
@@ -23,6 +23,14 @@
     msg: openshift_hosted_logging_use_journal is deprecated.  Fluentd will automatically detect which logging driver is being used.
   when: openshift_hosted_logging_use_journal is defined
 
+- fail:
+    msg: Invalid openshift_logging_mux_client_mode [{{ openshift_logging_mux_client_mode }}], one of {{ __allowed_mux_client_modes }} allowed
+  when: openshift_logging_mux_client_mode is defined and not openshift_logging_mux_client_mode in __allowed_mux_client_modes
+
+- debug:
+    msg: WARNING Use of openshift_logging_mux_client_mode=minimal is not recommended due to current scaling issues
+  when: openshift_logging_mux_client_mode is defined and openshift_logging_mux_client_mode == 'minimal'
+
 - include: determine_version.yaml
 
 # allow passing in a tempdir
diff --git a/roles/openshift_logging_fluentd/templates/fluentd.j2 b/roles/openshift_logging_fluentd/templates/fluentd.j2
index a4cf9a149..39dffba19 100644
--- a/roles/openshift_logging_fluentd/templates/fluentd.j2
+++ b/roles/openshift_logging_fluentd/templates/fluentd.j2
@@ -64,7 +64,7 @@ spec:
           readOnly: true
         - name: filebufferstorage
           mountPath: /var/lib/fluentd
-{% if openshift_logging_use_mux_client | bool %}
+{% if openshift_logging_mux_client_mode is defined %}
         - name: muxcerts
           mountPath: /etc/fluent/muxkeys
           readOnly: true
@@ -112,10 +112,12 @@ spec:
             resourceFieldRef:
               containerName: "{{ daemonset_container_name }}"
               resource: limits.memory
-        - name: "USE_MUX_CLIENT"
-          value: "{{ openshift_logging_use_mux_client | default('false') | lower }}"
         - name: "FILE_BUFFER_LIMIT"
           value: "{{ openshift_logging_fluentd_file_buffer_limit | default('1Gi') }}"
+{% if openshift_logging_mux_client_mode is defined %}
+        - name: "MUX_CLIENT_MODE"
+          value: "{{ openshift_logging_mux_client_mode }}"
+{% endif %}
       volumes:
       - name: runlogjournal
         hostPath:
@@ -144,7 +146,7 @@ spec:
       - name: dockerdaemoncfg
         hostPath:
           path: /etc/docker
-{% if openshift_logging_use_mux_client | bool %}
+{% if openshift_logging_mux_client_mode is defined %}
       - name: muxcerts
         secret:
           secretName: logging-mux
diff --git a/roles/openshift_logging_fluentd/vars/main.yml b/roles/openshift_logging_fluentd/vars/main.yml
index ad3fb0bdd..ec8e565c3 100644
--- a/roles/openshift_logging_fluentd/vars/main.yml
+++ b/roles/openshift_logging_fluentd/vars/main.yml
@@ -2,3 +2,4 @@
 __latest_fluentd_version: "3_5"
 __allowed_fluentd_versions: ["3_5", "3_6"]
 __allowed_fluentd_types: ["hosted", "secure-aggregator", "secure-host"]
+__allowed_mux_client_modes: ["minimal", "maximal"]
diff --git a/roles/openshift_logging_mux/defaults/main.yml b/roles/openshift_logging_mux/defaults/main.yml
index 35fc7146f..7a3da9b4c 100644
--- a/roles/openshift_logging_mux/defaults/main.yml
+++ b/roles/openshift_logging_mux/defaults/main.yml
@@ -28,6 +28,7 @@ openshift_logging_mux_journal_source: "{{ openshift_hosted_logging_journal_sourc
 openshift_logging_mux_journal_read_from_head: "{{ openshift_hosted_logging_journal_read_from_head | default('') }}"
 
 openshift_logging_mux_allow_external: False
+openshift_logging_use_mux: "{{ openshift_logging_mux_allow_external | default(False) }}"
 openshift_logging_mux_hostname: "{{ 'mux.' ~ (openshift_master_default_subdomain | default('router.default.svc.cluster.local', true)) }}"
 openshift_logging_mux_port: 24284
 # the namespace to use for undefined projects should come first, followed by any
diff --git a/roles/openshift_logging_mux/templates/mux.j2 b/roles/openshift_logging_mux/templates/mux.j2
index e43d9d397..70afe5cee 100644
--- a/roles/openshift_logging_mux/templates/mux.j2
+++ b/roles/openshift_logging_mux/templates/mux.j2
@@ -101,8 +101,6 @@ spec:
           value: "{{ openshift_logging_mux_port }}"
         - name: USE_MUX
           value: "true"
-        - name: MUX_ALLOW_EXTERNAL
-          value: "{{ openshift_logging_mux_allow_external | default('false') | lower }}"
         - name: "BUFFER_QUEUE_LIMIT"
           value: "{{ openshift_logging_mux_buffer_queue_limit }}"
         - name: "BUFFER_SIZE_LIMIT"