add fluentd logging driver config check

4 files changed, 376 insertions, 37 deletions

diff --git a/roles/openshift_health_checker/test/elasticsearch_test.py b/roles/openshift_health_checker/test/elasticsearch_test.py
index 9edfc17c7..67408609a 100644
--- a/roles/openshift_health_checker/test/elasticsearch_test.py
+++ b/roles/openshift_health_checker/test/elasticsearch_test.py
@@ -6,14 +6,6 @@ from openshift_checks.logging.elasticsearch import Elasticsearch
 task_vars_config_base = dict(openshift=dict(common=dict(config_base='/etc/origin')))
 
 
-def canned_elasticsearch(task_vars=None, exec_oc=None):
-    """Create an Elasticsearch check object with canned exec_oc method"""
-    check = Elasticsearch("dummy", task_vars or {})  # fails if a module is actually invoked
-    if exec_oc:
-        check._exec_oc = exec_oc
-    return check
-
-
 def assert_error(error, expect_error):
     if expect_error:
         assert error
@@ -50,10 +42,10 @@ split_es_pod = {
 
 
 def test_check_elasticsearch():
-    assert 'No logging Elasticsearch pods' in canned_elasticsearch().check_elasticsearch([])
+    assert 'No logging Elasticsearch pods' in Elasticsearch().check_elasticsearch([])
 
     # canned oc responses to match so all the checks pass
-    def _exec_oc(cmd, args):
+    def _exec_oc(ns, cmd, args):
         if '_cat/master' in cmd:
             return 'name logging-es'
         elif '/_nodes' in cmd:
@@ -65,7 +57,9 @@ def test_check_elasticsearch():
         else:
             raise Exception(cmd)
 
-    assert not canned_elasticsearch({}, _exec_oc).check_elasticsearch([plain_es_pod])
+    check = Elasticsearch(None, {})
+    check.exec_oc = _exec_oc
+    assert not check.check_elasticsearch([plain_es_pod])
 
 
 def pods_by_name(pods):
@@ -88,8 +82,8 @@ def pods_by_name(pods):
 ])
 def test_check_elasticsearch_masters(pods, expect_error):
     test_pods = list(pods)
-    check = canned_elasticsearch(task_vars_config_base, lambda cmd, args: test_pods.pop(0)['_test_master_name_str'])
-
+    check = Elasticsearch(None, task_vars_config_base)
+    check.execute_module = lambda cmd, args: {'result': test_pods.pop(0)['_test_master_name_str']}
     errors = check._check_elasticsearch_masters(pods_by_name(pods))
     assert_error(''.join(errors), expect_error)
 
@@ -124,7 +118,8 @@ es_node_list = {
     ),
 ])
 def test_check_elasticsearch_node_list(pods, node_list, expect_error):
-    check = canned_elasticsearch(task_vars_config_base, lambda cmd, args: json.dumps(node_list))
+    check = Elasticsearch(None, task_vars_config_base)
+    check.execute_module = lambda cmd, args: {'result': json.dumps(node_list)}
 
     errors = check._check_elasticsearch_node_list(pods_by_name(pods))
     assert_error(''.join(errors), expect_error)
@@ -149,7 +144,8 @@ def test_check_elasticsearch_node_list(pods, node_list, expect_error):
 ])
 def test_check_elasticsearch_cluster_health(pods, health_data, expect_error):
     test_health_data = list(health_data)
-    check = canned_elasticsearch(task_vars_config_base, lambda cmd, args: json.dumps(test_health_data.pop(0)))
+    check = Elasticsearch(None, task_vars_config_base)
+    check.execute_module = lambda cmd, args: {'result': json.dumps(test_health_data.pop(0))}
 
     errors = check._check_es_cluster_health(pods_by_name(pods))
     assert_error(''.join(errors), expect_error)
@@ -174,7 +170,8 @@ def test_check_elasticsearch_cluster_health(pods, health_data, expect_error):
     ),
 ])
 def test_check_elasticsearch_diskspace(disk_data, expect_error):
-    check = canned_elasticsearch(task_vars_config_base, lambda cmd, args: disk_data)
+    check = Elasticsearch(None, task_vars_config_base)
+    check.execute_module = lambda cmd, args: {'result': disk_data}
 
     errors = check._check_elasticsearch_diskspace(pods_by_name([plain_es_pod]))
     assert_error(''.join(errors), expect_error)
diff --git a/roles/openshift_health_checker/test/fluentd_config_test.py b/roles/openshift_health_checker/test/fluentd_config_test.py
new file mode 100644
index 000000000..8a2d8b72b
--- /dev/null
+++ b/roles/openshift_health_checker/test/fluentd_config_test.py
@@ -0,0 +1,357 @@
+import pytest
+
+from openshift_checks.logging.fluentd_config import FluentdConfig, OpenShiftCheckException
+
+
+def canned_fluentd_pod(containers):
+    return {
+        "metadata": {
+            "labels": {"component": "fluentd", "deploymentconfig": "logging-fluentd"},
+            "name": "logging-fluentd-1",
+        },
+        "spec": {
+            "host": "node1",
+            "nodeName": "node1",
+            "containers": containers,
+        },
+        "status": {
+            "phase": "Running",
+            "containerStatuses": [{"ready": True}],
+            "conditions": [{"status": "True", "type": "Ready"}],
+        }
+    }
+
+
+fluentd_pod = {
+    "metadata": {
+        "labels": {"component": "fluentd", "deploymentconfig": "logging-fluentd"},
+        "name": "logging-fluentd-1",
+    },
+    "spec": {
+        "host": "node1",
+        "nodeName": "node1",
+        "containers": [
+            {
+                "name": "container1",
+                "env": [
+                    {
+                        "name": "USE_JOURNAL",
+                        "value": "true",
+                    }
+                ],
+            }
+        ],
+    },
+    "status": {
+        "phase": "Running",
+        "containerStatuses": [{"ready": True}],
+        "conditions": [{"status": "True", "type": "Ready"}],
+    }
+}
+
+not_running_fluentd_pod = {
+    "metadata": {
+        "labels": {"component": "fluentd", "deploymentconfig": "logging-fluentd"},
+        "name": "logging-fluentd-2",
+    },
+    "status": {
+        "phase": "Unknown",
+        "containerStatuses": [{"ready": True}, {"ready": False}],
+        "conditions": [{"status": "True", "type": "Ready"}],
+    }
+}
+
+
+@pytest.mark.parametrize('name, use_journald, logging_driver, extra_words', [
+    (
+        'test success with use_journald=false, and docker config set to use "json-file"',
+        False,
+        "json-file",
+        [],
+    ),
+], ids=lambda argvals: argvals[0])
+def test_check_logging_config_non_master(name, use_journald, logging_driver, extra_words):
+    def execute_module(module_name, args):
+        if module_name == "docker_info":
+            return {
+                "info": {
+                    "LoggingDriver": logging_driver,
+                }
+            }
+
+        return {}
+
+    task_vars = dict(
+        group_names=["nodes", "etcd"],
+        openshift_logging_fluentd_use_journal=use_journald,
+        openshift=dict(
+            common=dict(config_base=""),
+        ),
+    )
+
+    check = FluentdConfig(execute_module, task_vars)
+    check.execute_module = execute_module
+    error = check.check_logging_config()
+
+    assert error is None
+
+
+@pytest.mark.parametrize('name, use_journald, logging_driver, words', [
+    (
+        'test failure with use_journald=false, but docker config set to use "journald"',
+        False,
+        "journald",
+        ['json log files', 'has been set to use "journald"'],
+    ),
+    (
+        'test failure with use_journald=false, but docker config set to use an "unsupported" driver',
+        False,
+        "unsupported",
+        ["json log files", 'has been set to use "unsupported"'],
+    ),
+    (
+        'test failure with use_journald=true, but docker config set to use "json-file"',
+        True,
+        "json-file",
+        ['logs from "journald"', 'has been set to use "json-file"'],
+    ),
+], ids=lambda argvals: argvals[0])
+def test_check_logging_config_non_master_failed(name, use_journald, logging_driver, words):
+    def execute_module(module_name, args):
+        if module_name == "docker_info":
+            return {
+                "info": {
+                    "LoggingDriver": logging_driver,
+                }
+            }
+
+        return {}
+
+    task_vars = dict(
+        group_names=["nodes", "etcd"],
+        openshift_logging_fluentd_use_journal=use_journald,
+        openshift=dict(
+            common=dict(config_base=""),
+        ),
+    )
+
+    check = FluentdConfig(execute_module, task_vars)
+    check.execute_module = execute_module
+    error = check.check_logging_config()
+
+    assert error is not None
+    for word in words:
+        assert word in error
+
+
+@pytest.mark.parametrize('name, pods, logging_driver, extra_words', [
+    # use_journald returns false (not using journald), but check succeeds
+    # since docker is set to use json-file
+    (
+        'test success with use_journald=false, and docker config set to use default driver "json-file"',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [{
+                        "name": "USE_JOURNAL",
+                        "value": "false",
+                    }],
+                },
+            ]
+        )],
+        "json-file",
+        [],
+    ),
+    (
+        'test success with USE_JOURNAL env var missing and docker config set to use default driver "json-file"',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [{
+                        "name": "RANDOM",
+                        "value": "value",
+                    }],
+                },
+            ]
+        )],
+        "json-file",
+        [],
+    ),
+], ids=lambda argvals: argvals[0])
+def test_check_logging_config_master(name, pods, logging_driver, extra_words):
+    def execute_module(module_name, args):
+        if module_name == "docker_info":
+            return {
+                "info": {
+                    "LoggingDriver": logging_driver,
+                }
+            }
+
+        return {}
+
+    task_vars = dict(
+        group_names=["masters"],
+        openshift=dict(
+            common=dict(config_base=""),
+        ),
+    )
+
+    def get_pods(namespace, logging_component):
+        return pods, None
+
+    check = FluentdConfig(execute_module, task_vars)
+    check.execute_module = execute_module
+    check.get_pods_for_component = get_pods
+    error = check.check_logging_config()
+
+    assert error is None
+
+
+@pytest.mark.parametrize('name, pods, logging_driver, words', [
+    (
+        'test failure with use_journald=false, but docker config set to use "journald"',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [{
+                        "name": "USE_JOURNAL",
+                        "value": "false",
+                    }],
+                },
+            ]
+        )],
+        "journald",
+        ['json log files', 'has been set to use "journald"'],
+    ),
+    (
+        'test failure with use_journald=true, but docker config set to use "json-file"',
+        [fluentd_pod],
+        "json-file",
+        ['logs from "journald"', 'has been set to use "json-file"'],
+    ),
+    (
+        'test failure with use_journald=false, but docker set to use an "unsupported" driver',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [{
+                        "name": "USE_JOURNAL",
+                        "value": "false",
+                    }],
+                },
+            ]
+        )],
+        "unsupported",
+        ["json log files", 'has been set to use "unsupported"'],
+    ),
+    (
+        'test failure with USE_JOURNAL env var missing and docker config set to use "journald"',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [{
+                        "name": "RANDOM",
+                        "value": "value",
+                    }],
+                },
+            ]
+        )],
+        "journald",
+        ["configuration is set to", "json log files"],
+    ),
+], ids=lambda argvals: argvals[0])
+def test_check_logging_config_master_failed(name, pods, logging_driver, words):
+    def execute_module(module_name, args):
+        if module_name == "docker_info":
+            return {
+                "info": {
+                    "LoggingDriver": logging_driver,
+                }
+            }
+
+        return {}
+
+    task_vars = dict(
+        group_names=["masters"],
+        openshift=dict(
+            common=dict(config_base=""),
+        ),
+    )
+
+    def get_pods(namespace, logging_component):
+        return pods, None
+
+    check = FluentdConfig(execute_module, task_vars)
+    check.execute_module = execute_module
+    check.get_pods_for_component = get_pods
+    error = check.check_logging_config()
+
+    assert error is not None
+    for word in words:
+        assert word in error
+
+
+@pytest.mark.parametrize('name, pods, response, logging_driver, extra_words', [
+    (
+        'test OpenShiftCheckException with no running containers',
+        [canned_fluentd_pod([])],
+        {
+            "failed": True,
+            "result": "unexpected",
+        },
+        "json-file",
+        ['no running containers'],
+    ),
+    (
+        'test OpenShiftCheckException one container and no env vars set',
+        [canned_fluentd_pod(
+            [
+                {
+                    "name": "container1",
+                    "env": [],
+                },
+            ]
+        )],
+        {
+            "failed": True,
+            "result": "unexpected",
+        },
+        "json-file",
+        ['no environment variables'],
+    ),
+], ids=lambda argvals: argvals[0])
+def test_check_logging_config_master_fails_on_unscheduled_deployment(name, pods, response, logging_driver, extra_words):
+    def execute_module(module_name, args):
+        if module_name == "docker_info":
+            return {
+                "info": {
+                    "LoggingDriver": logging_driver,
+                }
+            }
+
+        return {}
+
+    task_vars = dict(
+        group_names=["masters"],
+        openshift=dict(
+            common=dict(config_base=""),
+        ),
+    )
+
+    def get_pods(namespace, logging_component):
+        return pods, None
+
+    check = FluentdConfig(execute_module, task_vars)
+    check.get_pods_for_component = get_pods
+
+    with pytest.raises(OpenShiftCheckException) as error:
+        check.check_logging_config()
+
+    assert error is not None
+    for word in extra_words:
+        assert word in str(error)
diff --git a/roles/openshift_health_checker/test/fluentd_test.py b/roles/openshift_health_checker/test/fluentd_test.py
index 9cee57868..a84d89cef 100644
--- a/roles/openshift_health_checker/test/fluentd_test.py
+++ b/roles/openshift_health_checker/test/fluentd_test.py
@@ -4,14 +4,6 @@ import json
 from openshift_checks.logging.fluentd import Fluentd
 
 
-def canned_fluentd(exec_oc=None):
-    """Create a Fluentd check object with canned exec_oc method"""
-    check = Fluentd("dummy")  # fails if a module is actually invoked
-    if exec_oc:
-        check._exec_oc = exec_oc
-    return check
-
-
 def assert_error(error, expect_error):
     if expect_error:
         assert error
@@ -103,7 +95,7 @@ fluentd_node3_unlabeled = {
     ),
 ])
 def test_get_fluentd_pods(pods, nodes, expect_error):
-    check = canned_fluentd(exec_oc=lambda cmd, args: json.dumps(dict(items=nodes)))
-
+    check = Fluentd()
+    check.exec_oc = lambda ns, cmd, args: json.dumps(dict(items=nodes))
     error = check.check_fluentd(pods)
     assert_error(error, expect_error)
diff --git a/roles/openshift_health_checker/test/kibana_test.py b/roles/openshift_health_checker/test/kibana_test.py
index 3a880d300..0bf492511 100644
--- a/roles/openshift_health_checker/test/kibana_test.py
+++ b/roles/openshift_health_checker/test/kibana_test.py
@@ -11,14 +11,6 @@ except ImportError:
 from openshift_checks.logging.kibana import Kibana
 
 
-def canned_kibana(exec_oc=None):
-    """Create a Kibana check object with canned exec_oc method"""
-    check = Kibana()  # fails if a module is actually invoked
-    if exec_oc:
-        check._exec_oc = exec_oc
-    return check
-
-
 def assert_error(error, expect_error):
     if expect_error:
         assert error
@@ -68,7 +60,7 @@ not_running_kibana_pod = {
     ),
 ])
 def test_check_kibana(pods, expect_error):
-    check = canned_kibana()
+    check = Kibana()
     error = check.check_kibana(pods)
     assert_error(error, expect_error)
 
@@ -137,7 +129,8 @@ def test_check_kibana(pods, expect_error):
     ),
 ])
 def test_get_kibana_url(route, expect_url, expect_error):
-    check = canned_kibana(exec_oc=lambda cmd, args: json.dumps(route) if route else "")
+    check = Kibana()
+    check.exec_oc = lambda ns, cmd, args: json.dumps(route) if route else ""
 
     url, error = check._get_kibana_url()
     if expect_url:
@@ -210,7 +203,7 @@ def test_verify_url_external_failure(lib_result, expect, monkeypatch):
         raise lib_result
     monkeypatch.setattr(urllib2, 'urlopen', urlopen)
 
-    check = canned_kibana()
+    check = Kibana()
     check._get_kibana_url = lambda: ('url', None)
     check._verify_url_internal = lambda url: None