summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAndrew Butcher <abutcher@redhat.com>2016-01-12 16:39:06 -0500
committerAndrew Butcher <abutcher@redhat.com>2016-01-12 16:39:06 -0500
commit97be5890e2a34036a22d2d1e2586c83009ae6064 (patch)
treec52d7a106680601fec959154a7a80fae5f4735b9
parentf9aaa8ac13adf841823f35be594641bdc2ebecac (diff)
downloadopenshift-97be5890e2a34036a22d2d1e2586c83009ae6064.tar.gz
openshift-97be5890e2a34036a22d2d1e2586c83009ae6064.tar.bz2
openshift-97be5890e2a34036a22d2d1e2586c83009ae6064.tar.xz
openshift-97be5890e2a34036a22d2d1e2586c83009ae6064.zip
Validate pacemaker cluster members.
-rw-r--r--filter_plugins/openshift_master.py28
-rw-r--r--playbooks/common/openshift-master/restart.yml13
-rw-r--r--playbooks/common/openshift-master/restart_hosts.yml15
-rw-r--r--roles/openshift_facts/tasks/main.yml1
4 files changed, 54 insertions, 3 deletions
diff --git a/filter_plugins/openshift_master.py b/filter_plugins/openshift_master.py
index 8d7c62ad1..7ababc51a 100644
--- a/filter_plugins/openshift_master.py
+++ b/filter_plugins/openshift_master.py
@@ -463,6 +463,32 @@ class FilterModule(object):
IdentityProviderBase.validate_idp_list(idp_list)
return yaml.safe_dump([idp.to_dict() for idp in idp_list], default_flow_style=False)
+ @staticmethod
+ def validate_pcs_cluster(data, masters=None):
+ ''' Validates output from "pcs status", ensuring that each master
+ provided is online.
+ Ex: data = ('...',
+ 'PCSD Status:',
+ 'master1.example.com: Online',
+ 'master2.example.com: Online',
+ 'master3.example.com: Online',
+ '...')
+ masters = ['master1.example.com',
+ 'master2.example.com',
+ 'master3.example.com']
+ returns True
+ '''
+ if not issubclass(type(data), str):
+ raise errors.AnsibleFilterError("|failed expects data is a string")
+ if not issubclass(type(masters), list):
+ raise errors.AnsibleFilterError("|failed expects masters is a list")
+ valid = True
+ for master in masters:
+ if "{0}: Online".format(master) not in data:
+ valid = False
+ return valid
+
def filters(self):
''' returns a mapping of filters to methods '''
- return {"translate_idps": self.translate_idps}
+ return {"translate_idps": self.translate_idps,
+ "validate_pcs_cluster": self.validate_pcs_cluster}
diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml
index 7603f0d61..fa13a64cb 100644
--- a/playbooks/common/openshift-master/restart.yml
+++ b/playbooks/common/openshift-master/restart.yml
@@ -73,6 +73,7 @@
register: active_check_output
when: openshift.master.cluster_method == 'pacemaker'
failed_when: active_check_output.stdout not in ['active', 'inactive']
+ changed_when: false
- set_fact:
is_active: "{{ active_check_output.stdout == 'active' }}"
when: openshift.master.cluster_method == 'pacemaker'
@@ -98,6 +99,18 @@
with_items: "{{ groups.oo_masters_to_config | default([]) }}"
when: (hostvars[item]['current_host'] | default(false)) | bool
+- name: Validate pacemaker cluster
+ hosts: oo_active_masters
+ tasks:
+ - name: Retrieve pcs status
+ command: pcs status
+ register: pcs_status_output
+ changed_when: false
+ - fail:
+ msg: >
+ Pacemaker cluster validation failed. One or more nodes are not online.
+ when: not (pcs_status_output.stdout | validate_pcs_cluster(groups.oo_masters_to_config)) | bool
+
- name: Restart masters
hosts: oo_masters_to_config:!oo_active_masters:!oo_current_masters
vars:
diff --git a/playbooks/common/openshift-master/restart_hosts.yml b/playbooks/common/openshift-master/restart_hosts.yml
index 598e1ad63..ff206f5a2 100644
--- a/playbooks/common/openshift-master/restart_hosts.yml
+++ b/playbooks/common/openshift-master/restart_hosts.yml
@@ -16,8 +16,6 @@
delay=10
port="{{ openshift.master.api_port }}"
when: openshift.master.cluster_method != 'pacemaker'
-# When cluster_method is pacemaker we can only ensure that the host
-# restarted successfully.
- name: Wait for master to start
become: no
local_action:
@@ -25,4 +23,17 @@
host="{{ inventory_hostname }}"
state=started
delay=10
+ port=22
when: openshift.master.cluster_method == 'pacemaker'
+- name: Wait for master to become available
+ command: pcs status
+ register: pcs_status_output
+ until: pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname]) | bool
+ retries: 15
+ delay: 2
+ changed_when: false
+ when: openshift.master.cluster_method == 'pacemaker'
+- fail:
+ msg: >
+ Pacemaker cluster validation failed {{ inventory hostname }} is not online.
+ when: openshift.master.cluster_method == 'pacemaker' and not (pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname])) | bool
diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml
index 87fa99a3b..e40a1b329 100644
--- a/roles/openshift_facts/tasks/main.yml
+++ b/roles/openshift_facts/tasks/main.yml
@@ -10,6 +10,7 @@
shell: ls /run/ostree-booted
ignore_errors: yes
failed_when: false
+ changed_when: false
register: ostree_output
# Locally setup containerized facts for now