summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--library/openshift_cert_expiry.py428
-rw-r--r--playbooks/byo/openshift-cluster/check-cert-expiry.yaml35
-rw-r--r--playbooks/common/openshift-cluster/check-cert-expiry.yaml37
-rw-r--r--utils/Makefile12
-rw-r--r--utils/test-requirements.txt1
5 files changed, 510 insertions, 3 deletions
diff --git a/library/openshift_cert_expiry.py b/library/openshift_cert_expiry.py
new file mode 100644
index 000000000..cd8662f67
--- /dev/null
+++ b/library/openshift_cert_expiry.py
@@ -0,0 +1,428 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# etcd config file
+import ConfigParser
+# Expiration parsing
+import datetime
+# File path stuff
+import os
+# Config file parsing
+import yaml
+# Certificate loading
+import OpenSSL.crypto
+
+
+DOCUMENTATION = '''
+---
+module: openshift_cert_expiry
+short_description: Check OpenShift Container Platform (OCP) and Kube certificate expirations on a cluster
+description:
+ - The M(openshift_cert_expiry) module has two basic functions: to flag certificates which will expire in a set window of time from now, and to notify you about certificates which have already expired.
+ - When the module finishes, a summary of the examination is returned. Each certificate in the summary has a C(health) key with a value of one of the following:
+ - C(ok) - not expired, and outside of the expiration C(warning_days) window.
+ - C(warning) - not expired, but will expire between now and the C(warning_days) window.
+ - C(expired) - an expired certificate.
+ - Certificate flagging follow this logic:
+ - If the expiration date is before now then the certificate is classified as C(expired).
+ - The certificates time to live (expiration date - now) is calculated, if that time window is less than C(warning_days) the certificate is classified as C(warning).
+ - All other conditions are classified as C(ok).
+ - The following keys are ALSO present in the certificate summary:
+ - C(cert_cn) - The common name of the certificate (additional CNs present in SAN extensions are omitted)
+ - C(days_remaining) - The number of days until the certificate expires.
+ - C(expiry) - The date the certificate expires on.
+ - C(path) - The full path to the certificate on the examined host.
+version_added: "0.0"
+options:
+ config_base:
+ description:
+ - Base path to OCP system settings.
+ required: false
+ default: /etc/origin
+ warning_days:
+ description:
+ - Flag certificates which will expire in C(warning_days) days from now.
+ required: false
+ default: 30
+ show_all:
+ description:
+ - Enable this option to show analysis of ALL certificates examined by this module.
+ - By default only certificates which have expired, or will expire within the C(warning_days) window will be reported.
+ required: false
+ default: false
+
+author: "Tim Bielawa (@tbielawa) <tbielawa@redhat.com>"
+'''
+
+EXAMPLES = '''
+# Default invocation, only notify about expired certificates or certificates which will expire within 30 days from now
+- openshift_cert_expiry:
+
+# Expand the warning window to show certificates expiring within a year from now
+- openshift_cert_expiry: warning_days=365
+
+# Show expired, soon to expire (now + 30 days), and all other certificates examined
+- openshift_cert_expiry: show_all=true
+'''
+
+
+######################################################################
+# etcd does not begin their config file with an opening [section] as
+# required by the Python ConfigParser module. We hack around it by
+# slipping one in ourselves prior to parsing.
+#
+# Source: Alex Martelli - http://stackoverflow.com/a/2819788/6490583
+class FakeSecHead(object):
+ def __init__(self, fp):
+ self.fp = fp
+ self.sechead = '[ETCD]\n'
+
+ def readline(self):
+ if self.sechead:
+ try:
+ return self.sechead
+ finally:
+ self.sechead = None
+ else:
+ return self.fp.readline()
+
+######################################################################
+
+def filter_paths(path_list):
+ # `path_list` - A list of file paths to check. Only files which
+ # exist will be returned
+ return filter(
+ lambda p: os.path.exists(os.path.realpath(p)),
+ path_list)
+
+def load_and_handle_cert(cert_string, now, base64decode=False):
+ """Load a certificate, split off the good parts, and return some
+useful data
+
+Params:
+
+- `cert_string` (string) - a certificate loaded into a string object
+- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against
+- `base64decode` (bool) - run .decode('base64') on the input?
+
+Returns:
+A 3-tuple of the form: (certificate_common_name, certificate_expiry_date, certificate_time_remaining)
+
+ """
+ if base64decode:
+ _cert_string = cert_string.decode('base-64')
+ else:
+ _cert_string = cert_string
+
+ cert_loaded = OpenSSL.crypto.load_certificate(
+ OpenSSL.crypto.FILETYPE_PEM, _cert_string)
+
+ # Strip the subject down to just the value of the first name
+ cert_subject = cert_loaded.get_subject().get_components()[0][1]
+
+ # Grab the expiration date
+ cert_expiry = cert_loaded.get_notAfter()
+ cert_expiry_date = datetime.datetime.strptime(
+ cert_expiry,
+ # example get_notAfter() => 20180922170439Z
+ '%Y%m%d%H%M%SZ')
+
+ time_remaining = cert_expiry_date - now
+
+ return (cert_subject, cert_expiry_date, time_remaining)
+
+def classify_cert(cert_meta, now, time_remaining, expire_window, cert_list):
+ """Given metadata about a certificate under examination, classify it
+ into one of three categories, 'ok', 'warning', and 'expired'.
+
+Params:
+
+- `cert_meta` dict - A dict with certificate metadata. Required fields
+ include: 'cert_cn', 'path', 'expiry', 'days_remaining', 'health'.
+- `now` (datetime) - a datetime object of the time to calculate the certificate 'time_remaining' against
+- `time_remaining` (datetime.timedelta) - a timedelta for how long until the cert expires
+- `expire_window` (datetime.timedelta) - a timedelta for how long the warning window is
+- `cert_list` list - A list to shove the classified cert into
+
+Return:
+- `cert_list` - The updated list of classified certificates
+ """
+ expiry_str = str(cert_meta['expiry'])
+ # Categorization
+ if cert_meta['expiry'] < now:
+ # This already expired, must NOTIFY
+ cert_meta['health'] = 'expired'
+ elif time_remaining < expire_window:
+ # WARN about this upcoming expirations
+ cert_meta['health'] = 'warning'
+ else:
+ # Not expired or about to expire
+ cert_meta['health'] = 'ok'
+
+ cert_meta['expiry'] = expiry_str
+ cert_list.append(cert_meta)
+ return cert_list
+
+def tabulate_summary(certificates, kubeconfigs):
+ """Calculate the summary text for when the module finishes
+running. This includes counds of each classification and what have
+you.
+
+Params:
+
+- `certificates` (list of dicts) - Processed `expire_check_result`
+ dicts with filled in `health` keys for system certificates.
+- `kubeconfigs` (list of dicts) - Processed `expire_check_result`
+ dicts with filled in `health` keys for embedded kubeconfig
+ certificates.
+
+Return:
+- `summary_results` (dict) - Counts of each cert/kubeconfig
+ classification and total items examined.
+ """
+ summary_results = {
+ 'system_certificates': len(certificates),
+ 'kubeconfig_certificates': len(kubeconfigs),
+ 'total': len(certificates + kubeconfigs),
+ 'ok': 0,
+ 'warning': 0,
+ 'expired': 0
+ }
+
+ items = certificates + kubeconfigs
+ summary_results['expired'] = len([c for c in items if c['health'] == 'expired'])
+ summary_results['warning'] = len([c for c in items if c['health'] == 'warning'])
+ summary_results['ok'] = len([c for c in items if c['health'] == 'ok'])
+
+ return summary_results
+
+
+######################################################################
+def main():
+ module = AnsibleModule(
+ argument_spec=dict(
+ config_base=dict(
+ required=False,
+ default="/etc/origin",
+ type='str'),
+ warning_days=dict(
+ required=False,
+ default=int(30),
+ type='int'),
+ show_all=dict(
+ required=False,
+ default="False",
+ type='bool')
+ ),
+ supports_check_mode=True,
+ )
+
+ # Basic scaffolding for OpenShift spcific certs
+ openshift_base_config_path = module.params['config_base']
+ openshift_master_config_path = os.path.normpath(
+ os.path.join(openshift_base_config_path, "master/master-config.yaml")
+ )
+ openshift_node_config_path = os.path.normpath(
+ os.path.join(openshift_base_config_path, "node/node-config.yaml")
+ )
+ openshift_cert_check_paths = [
+ openshift_master_config_path,
+ openshift_node_config_path,
+ ]
+
+ # Paths for Kubeconfigs. Additional kubeconfigs are conditionally checked later in the code
+ kubeconfig_paths = [
+ os.path.normpath(
+ os.path.join(openshift_base_config_path, "master/admin.kubeconfig")
+ ),
+ os.path.normpath(
+ os.path.join(openshift_base_config_path, "master/openshift-master.kubeconfig")
+ ),
+ os.path.normpath(
+ os.path.join(openshift_base_config_path, "master/openshift-node.kubeconfig")
+ ),
+ os.path.normpath(
+ os.path.join(openshift_base_config_path, "master/openshift-router.kubeconfig")
+ ),
+ ]
+
+ # Expiry checking stuff
+ now = datetime.datetime.now()
+ # todo, catch exception for invalid input and return a fail_json
+ warning_days = int(module.params['warning_days'])
+ expire_window = datetime.timedelta(days=warning_days)
+
+ # Module stuff
+ #
+ # The results of our cert checking to return from the task call
+ check_results = {}
+ check_results['meta'] = {}
+ check_results['meta']['warning_days'] = warning_days
+ check_results['meta']['checked_at_time'] = str(now)
+ check_results['meta']['warn_after_date'] = str(now + expire_window)
+ check_results['meta']['show_all'] = str(module.params['show_all'])
+ # All the analyzed certs accumulate here
+ certs = []
+
+ ######################################################################
+ # Sure, why not? Let's enable check mode.
+ if module.check_mode:
+ check_results['certs'] = []
+ module.exit_json(
+ check_results=check_results,
+ msg="Checked 0 certificates. Expired/Warning/OK: 0/0/0. Warning window: %s days" % module.params['warning_days'],
+ rc=0,
+ changed=False
+ )
+
+ ######################################################################
+ # Check for OpenShift Container Platform specific certs
+ ######################################################################
+ for os_cert in filter_paths(openshift_cert_check_paths):
+ # Open up that config file and locate the cert and CA
+ with open(os_cert, 'r') as fp:
+ cert_meta = {}
+ cfg = yaml.load(fp)
+ # cert files are specified in parsed `fp` as relative to the path
+ # of the original config file. 'master-config.yaml' with certFile
+ # = 'foo.crt' implies that 'foo.crt' is in the same
+ # directory. certFile = '../foo.crt' is in the parent directory.
+ cfg_path = os.path.dirname(fp.name)
+ cert_meta['certFile'] = os.path.join(cfg_path, cfg['servingInfo']['certFile'])
+ cert_meta['clientCA'] = os.path.join(cfg_path, cfg['servingInfo']['clientCA'])
+
+ ######################################################################
+ # Load the certificate and the CA, parse their expiration dates into
+ # datetime objects so we can manipulate them later
+ for _, v in cert_meta.iteritems():
+ with open(v, 'r') as fp:
+ cert = fp.read()
+ cert_subject, cert_expiry_date, time_remaining = load_and_handle_cert(cert, now)
+
+ expire_check_result = {
+ 'cert_cn': cert_subject,
+ 'path': fp.name,
+ 'expiry': cert_expiry_date,
+ 'days_remaining': time_remaining.days,
+ 'health': None,
+ }
+
+ classify_cert(expire_check_result, now, time_remaining, expire_window, certs)
+
+ ######################################################################
+ # /Check for OpenShift Container Platform specific certs
+ ######################################################################
+
+ ######################################################################
+ # Check service Kubeconfigs
+ ######################################################################
+ kubeconfigs = []
+
+ # There may be additional kubeconfigs to check, but their naming
+ # is less predictable than the ones we've already assembled.
+
+ try:
+ # Try to read the standard 'node-config.yaml' file to check if
+ # this host is a node.
+ with open(openshift_node_config_path, 'r') as fp:
+ cfg = yaml.load(fp)
+ # OK, the config file exists, therefore this is a
+ # node. Nodes have their own kubeconfig files to
+ # communicate with the master API. Let's read the relative
+ # path to that file from the node config.
+ node_masterKubeConfig = cfg['masterKubeConfig']
+ # As before, the path to the 'masterKubeConfig' file is
+ # relative to `fp`
+ cfg_path = os.path.dirname(fp.name)
+ node_kubeconfig = os.path.join(cfg_path, node_masterKubeConfig)
+ with open(node_kubeconfig, 'r') as fp:
+ # Read in the nodes kubeconfig file and grab the good stuff
+ cfg = yaml.load(fp)
+ c = cfg['users'][0]['user']['client-certificate-data']
+ (cert_subject,
+ cert_expiry_date,
+ time_remaining) = load_and_handle_cert(c, now, base64decode=True)
+
+ expire_check_result = {
+ 'cert_cn': cert_subject,
+ 'path': fp.name,
+ 'expiry': cert_expiry_date,
+ 'days_remaining': time_remaining.days,
+ 'health': None,
+ }
+
+ classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs)
+ except Exception:
+ # This is not a node
+ pass
+
+ for kube in filter_paths(kubeconfig_paths):
+ with open(kube, 'r') as fp:
+ # TODO: Maybe consider catching exceptions here?
+ cfg = yaml.load(fp)
+ # Per conversation, "the kubeconfigs you care about:
+ # admin, router, registry should all be single
+ # value". Following that advice we only grab the data for
+ # the user at index 0 in the 'users' list. There should
+ # not be more than one user.
+ c = cfg['users'][0]['user']['client-certificate-data']
+ (cert_subject,
+ cert_expiry_date,
+ time_remaining) = load_and_handle_cert(c, now, base64decode=True)
+
+ expire_check_result = {
+ 'cert_cn': cert_subject,
+ 'path': fp.name,
+ 'expiry': cert_expiry_date,
+ 'days_remaining': time_remaining.days,
+ 'health': None,
+ }
+
+ classify_cert(expire_check_result, now, time_remaining, expire_window, kubeconfigs)
+
+
+ ######################################################################
+ # /Check service Kubeconfigs
+ ######################################################################
+ res = tabulate_summary(certs, kubeconfigs)
+
+ msg = "Checked {count} certificates and kubeconfigs. Expired/Warning/OK: {exp}/{warn}/{ok}. Warning window: {window} days".format(
+ count=res['total'],
+ exp=res['expired'],
+ warn=res['warning'],
+ ok=res['ok'],
+ window=int(module.params['warning_days']),
+ )
+
+ # By default we only return detailed information about expired or
+ # warning certificates. If show_all is true then we will print all
+ # the certificates examined.
+ if not module.params['show_all']:
+ check_results['certs'] = filter(lambda ctr: ctr['health'] in ['expired', 'warning'], certs)
+ check_results['kubeconfigs'] = filter(lambda ctr: ctr['health'] in ['expired', 'warning'], kubeconfigs)
+ else:
+ check_results['certs'] = certs
+ check_results['kubeconfigs'] = kubeconfigs
+
+ # Sort the final results to report in order of ascending safety
+ # time. That is to say, the certificates which will expire sooner
+ # will be at the front of the list and certificates which will
+ # expire later are at the end.
+ check_results['certs'] = sorted(check_results['certs'], cmp=lambda x, y: cmp(x['days_remaining'], y['days_remaining']))
+ check_results['kubeconfigs'] = sorted(check_results['kubeconfigs'], cmp=lambda x, y: cmp(x['days_remaining'], y['days_remaining']))
+ # This module will never change anything, but we might want to
+ # change the return code parameter if there is some catastrophic
+ # error we noticed earlier
+ module.exit_json(
+ check_results=check_results,
+ summary=res,
+ msg=msg,
+ rc=0,
+ changed=False
+ )
+
+######################################################################
+# import module snippets
+from ansible.module_utils.basic import AnsibleModule
+if __name__ == '__main__':
+ main()
diff --git a/playbooks/byo/openshift-cluster/check-cert-expiry.yaml b/playbooks/byo/openshift-cluster/check-cert-expiry.yaml
new file mode 100644
index 000000000..39efdbd36
--- /dev/null
+++ b/playbooks/byo/openshift-cluster/check-cert-expiry.yaml
@@ -0,0 +1,35 @@
+---
+# check-cert-expiry.yaml - A utility for cluster ops to scan through
+# (critical) certificates for the ongoing operations of a cluster.
+
+# We do not support all Ansible versions. This is our safety net.
+- include: ../../common/openshift-cluster/verify_ansible_version.yml
+
+- name: Generate the l_oo_all_hosts group
+ hosts: localhost
+ connection: local
+ become: no
+ gather_facts: no
+ tasks:
+ - include_vars: cluster_hosts.yml
+ - add_host:
+ name: "{{ item }}"
+ groups: l_oo_all_hosts
+ with_items: "{{ g_all_hosts | default([]) }}"
+
+# This may seem redundant, running `include_vars` again on the list of
+# hosts in the group 'l_oo_all_hosts' which we just created. But the
+# fact of the matter is that if we don't re-run include_vars on the
+# new host group we created, then they will not have access to those
+# same group variables they were birthed from.
+#
+# Go ahead and try to 'debug: var=g_all_hosts' later on (without this
+# play) and you'll find that the result is UNDEFINED VARIABLE.
+- name: Inject cluster hosts variables into l_oo_all_hosts
+ hosts: l_oo_all_hosts
+ gather_facts: no
+ tasks:
+ - include_vars: cluster_hosts.yml
+
+# This is where the actual business gets started:
+- include: ../../common/openshift-cluster/check-cert-expiry.yaml
diff --git a/playbooks/common/openshift-cluster/check-cert-expiry.yaml b/playbooks/common/openshift-cluster/check-cert-expiry.yaml
new file mode 100644
index 000000000..e160383af
--- /dev/null
+++ b/playbooks/common/openshift-cluster/check-cert-expiry.yaml
@@ -0,0 +1,37 @@
+# ---
+# # This must be evaluated after cluster_hosts.yml has been evaluated on
+# # 'localhost' to generate l_oo_all_hosts and then subsequently
+# # evaluated again on the generated hosts group (l_oo_all_hosts).
+# - include: evaluate_groups.yml
+# tags:
+# - always
+
+# # Initialize cluster facts for oo_all_hosts using the openshift_facts
+# # role followed by the 'common' role
+# - include: initialize_facts.yml
+# tags:
+# - always
+
+# # Get the version to install from the first master, then synchronize
+# # that variable across all in oo_all_hosts
+# - include: initialize_openshift_version.yml
+# tags:
+# - always
+
+# # Earlier 'initialize_facts' included the openshift_facts role and
+# # used the openshift_facts module with the 'role' parameter set to
+# # 'common'. Now we're applying the openshift_facts role AGAIN but just
+# # to the subset of oo_all_hosts which require configuring.
+# - name: Load openshift_facts
+# hosts: oo_etcd_to_config:oo_masters_to_config:oo_nodes_to_config
+# roles:
+# - openshift_facts
+
+- hosts: all
+ become: yes
+ gather_facts: no
+ tasks:
+ - name: Check cert expirys on host
+ openshift_cert_expiry:
+ warning_days: 1500
+ show_all: true
diff --git a/utils/Makefile b/utils/Makefile
index 59aff92fd..bc708964b 100644
--- a/utils/Makefile
+++ b/utils/Makefile
@@ -31,6 +31,8 @@ ASCII2MAN = a2x -D $(dir $@) -d manpage -f manpage $<
MANPAGES := docs/man/man1/atomic-openshift-installer.1
VERSION := 1.3
+PEPEXCLUDES := E501,E121,E124
+
sdist: clean
python setup.py sdist
rm -fR $(SHORTNAME).egg-info
@@ -80,7 +82,7 @@ ci-pylint:
@echo "#############################################"
@echo "# Running PyLint Tests in virtualenv"
@echo "#############################################"
- . $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py ../callback_plugins/openshift_quick_installer.py
+ . $(NAME)env/bin/activate && python -m pylint --rcfile ../git/.pylintrc src/ooinstall/cli_installer.py src/ooinstall/oo_config.py src/ooinstall/openshift_ansible.py src/ooinstall/variants.py ../callback_plugins/openshift_quick_installer.py ../library/openshift_cert_expiry.py
ci-list-deps:
@echo "#############################################"
@@ -94,13 +96,17 @@ ci-pyflakes:
@echo "#################################################"
. $(NAME)env/bin/activate && pyflakes src/ooinstall/*.py
. $(NAME)env/bin/activate && pyflakes ../callback_plugins/openshift_quick_installer.py
+ . $(NAME)env/bin/activate && pyflakes ../library/openshift_cert_expiry.py
ci-pep8:
@echo "#############################################"
@echo "# Running PEP8 Compliance Tests in virtualenv"
@echo "#############################################"
- . $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 src/$(SHORTNAME)/
- . $(NAME)env/bin/activate && pep8 --ignore=E501,E121,E124 ../callback_plugins/openshift_quick_installer.py
+ . $(NAME)env/bin/activate && pep8 --ignore=$(PEPEXCLUDES) src/$(SHORTNAME)/
+ . $(NAME)env/bin/activate && pep8 --ignore=$(PEPEXCLUDES) ../callback_plugins/openshift_quick_installer.py
+# This one excludes E402 because it is an ansible module and the
+# boilerplate import statement is expected to be at the bottom
+ . $(NAME)env/bin/activate && pep8 --ignore=$(PEPEXCLUDES),E402 ../library/openshift_cert_expiry.py
ci: clean virtualenv ci-list-deps ci-pep8 ci-pylint ci-pyflakes ci-unittests
:
diff --git a/utils/test-requirements.txt b/utils/test-requirements.txt
index f2216a177..af91ab6a7 100644
--- a/utils/test-requirements.txt
+++ b/utils/test-requirements.txt
@@ -9,3 +9,4 @@ flake8
PyYAML
click
backports.functools_lru_cache
+pyOpenSSL