diff options
| -rw-r--r-- | filter_plugins/openshift_master.py | 28 | ||||
| -rw-r--r-- | playbooks/common/openshift-master/restart.yml | 13 | ||||
| -rw-r--r-- | playbooks/common/openshift-master/restart_hosts.yml | 15 | ||||
| -rw-r--r-- | roles/openshift_facts/tasks/main.yml | 1 |
4 files changed, 54 insertions, 3 deletions
diff --git a/filter_plugins/openshift_master.py b/filter_plugins/openshift_master.py index 8d7c62ad1..7ababc51a 100644 --- a/filter_plugins/openshift_master.py +++ b/filter_plugins/openshift_master.py @@ -463,6 +463,32 @@ class FilterModule(object): IdentityProviderBase.validate_idp_list(idp_list) return yaml.safe_dump([idp.to_dict() for idp in idp_list], default_flow_style=False) + @staticmethod + def validate_pcs_cluster(data, masters=None): + ''' Validates output from "pcs status", ensuring that each master + provided is online. + Ex: data = ('...', + 'PCSD Status:', + 'master1.example.com: Online', + 'master2.example.com: Online', + 'master3.example.com: Online', + '...') + masters = ['master1.example.com', + 'master2.example.com', + 'master3.example.com'] + returns True + ''' + if not issubclass(type(data), str): + raise errors.AnsibleFilterError("|failed expects data is a string") + if not issubclass(type(masters), list): + raise errors.AnsibleFilterError("|failed expects masters is a list") + valid = True + for master in masters: + if "{0}: Online".format(master) not in data: + valid = False + return valid + def filters(self): ''' returns a mapping of filters to methods ''' - return {"translate_idps": self.translate_idps} + return {"translate_idps": self.translate_idps, + "validate_pcs_cluster": self.validate_pcs_cluster} diff --git a/playbooks/common/openshift-master/restart.yml b/playbooks/common/openshift-master/restart.yml index 7603f0d61..fa13a64cb 100644 --- a/playbooks/common/openshift-master/restart.yml +++ b/playbooks/common/openshift-master/restart.yml @@ -73,6 +73,7 @@ register: active_check_output when: openshift.master.cluster_method == 'pacemaker' failed_when: active_check_output.stdout not in ['active', 'inactive'] + changed_when: false - set_fact: is_active: "{{ active_check_output.stdout == 'active' }}" when: openshift.master.cluster_method == 'pacemaker' @@ -98,6 +99,18 @@ with_items: "{{ groups.oo_masters_to_config | default([]) }}" when: (hostvars[item]['current_host'] | default(false)) | bool +- name: Validate pacemaker cluster + hosts: oo_active_masters + tasks: + - name: Retrieve pcs status + command: pcs status + register: pcs_status_output + changed_when: false + - fail: + msg: > + Pacemaker cluster validation failed. One or more nodes are not online. + when: not (pcs_status_output.stdout | validate_pcs_cluster(groups.oo_masters_to_config)) | bool + - name: Restart masters hosts: oo_masters_to_config:!oo_active_masters:!oo_current_masters vars: diff --git a/playbooks/common/openshift-master/restart_hosts.yml b/playbooks/common/openshift-master/restart_hosts.yml index 598e1ad63..ff206f5a2 100644 --- a/playbooks/common/openshift-master/restart_hosts.yml +++ b/playbooks/common/openshift-master/restart_hosts.yml @@ -16,8 +16,6 @@ delay=10 port="{{ openshift.master.api_port }}" when: openshift.master.cluster_method != 'pacemaker' -# When cluster_method is pacemaker we can only ensure that the host -# restarted successfully. - name: Wait for master to start become: no local_action: @@ -25,4 +23,17 @@ host="{{ inventory_hostname }}" state=started delay=10 + port=22 when: openshift.master.cluster_method == 'pacemaker' +- name: Wait for master to become available + command: pcs status + register: pcs_status_output + until: pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname]) | bool + retries: 15 + delay: 2 + changed_when: false + when: openshift.master.cluster_method == 'pacemaker' +- fail: + msg: > + Pacemaker cluster validation failed {{ inventory hostname }} is not online. + when: openshift.master.cluster_method == 'pacemaker' and not (pcs_status_output.stdout | validate_pcs_cluster([inventory_hostname])) | bool diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml index 87fa99a3b..e40a1b329 100644 --- a/roles/openshift_facts/tasks/main.yml +++ b/roles/openshift_facts/tasks/main.yml @@ -10,6 +10,7 @@ shell: ls /run/ostree-booted ignore_errors: yes failed_when: false + changed_when: false register: ostree_output # Locally setup containerized facts for now |