Initial HA master

- Ability to specify multiple masters
  - configures the CA only a single time on the first master
  - creates and distributes additional certs for additional master hosts

- Depending on the status of openshift_master_cluster_defer_ha (defaults to
  False) one of two actions are taken when multiple masters are defined
  1. If openshift_master_cluster_defer_ha is true
     a. Certs/configs for all masters are deployed
     b. openshift-master service is only started and enabled on the master
     c. HA configuration is expected to be handled by the user manually after
        the completion of the playbook run.
  2. If oepnshift_master_cluster_defer_ha is false or undefined
     a. Certs/configs for all masters are deployed
     b. a Pacemaker/RHEL HA cluster is configured
        i. VIPs are configured based on the values of
           openshift_master_cluster_vip and
           openshift_master_cluster_plublic_vip
        ii. The openshift-master service is configured as an active/passive
            cluster service

15 files changed, 234 insertions, 22 deletions

diff --git a/inventory/byo/hosts.example b/inventory/byo/hosts.example
index 41216dd89..56f4da5a2 100644
--- a/inventory/byo/hosts.example
+++ b/inventory/byo/hosts.example
@@ -38,14 +38,30 @@ deployment_type=enterprise
 # Allow all auth
 #openshift_master_identity_providers=[{'name': 'allow_all', 'login': 'true', 'challenge': 'true', 'kind': 'AllowAllPasswordIdentityProvider'}]
 
+# master cluster ha variables using pacemaker or RHEL HA
+#openshift_master_cluster_password=openshift_cluster
+#openshift_master_cluster_vip=192.168.133.25
+#openshift_master_cluster_public_vip=192.168.133.25
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+
+# master cluster ha variables when using a different HA solution
+# For installation the value of openshift_master_cluster_hostname must resolve
+# to the first master defined in the inventory.
+# The HA solution must be manually configured after installation and must ensure
+# that openshift-master is running on a single master host.
+#openshift_master_cluster_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_public_hostname=openshift-ansible.test.example.com
+#openshift_master_cluster_defer_ha=True
+
 # host group for masters
 [masters]
-ose3-master-ansible.test.example.com
+ose3-master[1:3]-ansible.test.example.com
 
 [etcd]
 ose3-etcd[1:3]-ansible.test.example.com
 
 # host group for nodes
 [nodes]
-ose3-master-ansible.test.example.com openshift_scheduleable=False
+ose3-master[1:3]-ansible.test.example.com openshift_scheduleable=False
 ose3-node[1:2]-ansible.test.example.com openshift_node_labels="{'region': 'primary', 'zone': 'default'}"
diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml
index 3956128e1..904ad2dab 100644
--- a/playbooks/common/openshift-master/config.yml
+++ b/playbooks/common/openshift-master/config.yml
@@ -27,6 +27,9 @@
           api_url: "{{ openshift_master_api_url | default(None) }}"
           api_use_ssl: "{{ openshift_master_api_use_ssl | default(None) }}"
           public_api_url: "{{ openshift_master_public_api_url | default(None) }}"
+          cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+          cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+          cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
           console_path: "{{ openshift_master_console_path | default(None) }}"
           console_port: "{{ openshift_master_console_port | default(None) }}"
           console_url: "{{ openshift_master_console_url | default(None) }}"
@@ -152,16 +155,26 @@
   roles:
   - openshift_master_certificates
   post_tasks:
+  - name: Remove generated etcd client certs when using external etcd
+    file:
+      path: "{{ master_generated_certs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+      state: absent
+    when: groups.oo_etcd_to_config is defined and groups.oo_etcd_to_config
+    with_nested:
+    - masters_needing_certs
+    - - master.etcd-client.crt
+      - master.etcd-client.key
+
   - name: Create a tarball of the master certs
     command: >
-      tar -czvf {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz
-        -C {{ master_generated_certs_dir }}/{{ item.master.cert_subdir }} .
+      tar -czvf {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz
+        -C {{ master_generated_certs_dir }}/{{ item.master_cert_subdir }} .
     args:
-      creates: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      creates: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
     with_items: masters_needing_certs
   - name: Retrieve the master cert tarball from the master
     fetch:
-      src: "{{ master_generated_certs_dir }}/{{ item.master.cert_subdir }}.tgz"
+      src: "{{ master_generated_certs_dir }}/{{ item.master_cert_subdir }}.tgz"
       dest: "{{ sync_tmpdir }}/"
       flat: yes
       fail_on_missing: yes
@@ -172,6 +185,7 @@
   hosts: oo_masters_to_config
   vars:
     sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}"
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
   pre_tasks:
   - name: Ensure certificate directory exists
     file:
@@ -192,9 +206,14 @@
     group_by: key=oo_masters_deployment_type_{{ openshift.common.deployment_type }}
     changed_when: False
 
-- name: Deploy OpenShift examples
+- name: Additional master configuration
   hosts: oo_first_master
+  vars:
+    openshift_master_ha: "{{ groups.oo_masters_to_config | length > 1 }}"
+    omc_cluster_hosts: "{{ groups.oo_masters_to_config | join(' ')}}"
   roles:
+  - role: openshift_master_cluster
+    when: openshift_master_ha | bool
   - openshift_examples
 
 # Additional instance config for online deployments
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 727861b07..d733639c3 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -367,9 +367,11 @@ def set_url_facts_if_unset(facts):
         console_path = facts['master']['console_path']
         etcd_use_ssl = facts['master']['etcd_use_ssl']
         etcd_hosts = facts['master']['etcd_hosts']
-        etcd_port = facts['master']['etcd_port'],
+        etcd_port = facts['master']['etcd_port']
         hostname = facts['common']['hostname']
         public_hostname = facts['common']['public_hostname']
+        cluster_hostname = facts['master'].get('cluster_hostname')
+        cluster_public_hostname = facts['master'].get('cluster_public_hostname')
 
         if 'etcd_urls' not in facts['master']:
             etcd_urls = []
@@ -384,24 +386,51 @@ def set_url_facts_if_unset(facts):
                                         etcd_port)]
             facts['master']['etcd_urls'] = etcd_urls
         if 'api_url' not in facts['master']:
-            facts['master']['api_url'] = format_url(api_use_ssl, hostname,
+            api_hostname = cluster_hostname if cluster_hostname else hostname
+            facts['master']['api_url'] = format_url(api_use_ssl, api_hostname,
                                                     api_port)
         if 'public_api_url' not in facts['master']:
+            api_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_api_url'] = format_url(api_use_ssl,
-                                                           public_hostname,
+                                                           api_public_hostname,
                                                            api_port)
         if 'console_url' not in facts['master']:
+            console_hostname = cluster_hostname if cluster_hostname else hostname
             facts['master']['console_url'] = format_url(console_use_ssl,
-                                                        hostname,
+                                                        console_hostname,
                                                         console_port,
                                                         console_path)
         if 'public_console_url' not in facts['master']:
+            console_public_hostname = cluster_public_hostname if cluster_public_hostname else public_hostname
             facts['master']['public_console_url'] = format_url(console_use_ssl,
-                                                               public_hostname,
+                                                               console_public_hostname,
                                                                console_port,
                                                                console_path)
     return facts
 
+def set_aggregate_facts(facts):
+    """ Set aggregate facts
+
+        Args:
+            facts (dict): existing facts
+        Returns:
+            dict: the facts dict updated with aggregated facts
+    """
+    all_hostnames = set()
+    if 'common' in facts:
+        all_hostnames.add(facts['common']['hostname'])
+        all_hostnames.add(facts['common']['public_hostname'])
+
+        if 'master' in facts:
+            if 'cluster_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_hostname'])
+            if 'cluster_public_hostname' in facts['master']:
+                all_hostnames.add(facts['master']['cluster_public_hostname'])
+
+        facts['common']['all_hostnames'] = list(all_hostnames)
+
+    return facts
+
 def set_sdn_facts_if_unset(facts):
     """ Set sdn facts if not already present in facts dict
 
@@ -675,6 +704,7 @@ class OpenShiftFacts(object):
         facts = set_identity_providers_if_unset(facts)
         facts = set_registry_url_if_unset(facts)
         facts = set_sdn_facts_if_unset(facts)
+        facts = set_aggregate_facts(facts)
         return dict(openshift=facts)
 
     def get_defaults(self, roles):
@@ -713,7 +743,7 @@ class OpenShiftFacts(object):
                           session_name='ssn', session_secrets_file='',
                           access_token_max_seconds=86400,
                           auth_token_max_seconds=500,
-                          oauth_grant_method='auto')
+                          oauth_grant_method='auto', cluster_defer_ha=False)
             defaults['master'] = master
 
         if 'node' in roles:
diff --git a/roles/openshift_master/defaults/main.yml b/roles/openshift_master/defaults/main.yml
index 11195e83e..ca8860099 100644
--- a/roles/openshift_master/defaults/main.yml
+++ b/roles/openshift_master/defaults/main.yml
@@ -15,6 +15,12 @@ os_firewall_allow:
   port: 24224/tcp
 - service: Fluentd td-agent udp
   port: 24224/udp
+- service: pcsd
+  port: 2224/tcp
+- service: Corosync UDP
+  port: 5404/udp
+- service: Corosync UDP
+  port: 5405/udp
 os_firewall_deny:
 - service: OpenShift api http
   port: 8080/tcp
diff --git a/roles/openshift_master/handlers/main.yml b/roles/openshift_master/handlers/main.yml
index 6fd4dfb51..d57f9a4ea 100644
--- a/roles/openshift_master/handlers/main.yml
+++ b/roles/openshift_master/handlers/main.yml
@@ -1,3 +1,4 @@
 ---
 - name: restart openshift-master
   service: name=openshift-master state=restarted
+  when: not openshift_master_ha
diff --git a/roles/openshift_master/tasks/main.yml b/roles/openshift_master/tasks/main.yml
index 02905f32d..2311568dd 100644
--- a/roles/openshift_master/tasks/main.yml
+++ b/roles/openshift_master/tasks/main.yml
@@ -8,6 +8,10 @@
     - openshift_master_oauth_grant_method in openshift_master_valid_grant_methods
   when: openshift_master_oauth_grant_method is defined
 
+- fail:
+    msg: "openshift_master_cluster_password must be set for multi-master installations"
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool and openshift_master_cluster_password is not defined
+
 - name: Install OpenShift Master package
   yum: pkg=openshift-master state=present
   register: install_result
@@ -16,6 +20,9 @@
   openshift_facts:
     role: master
     local_facts:
+      cluster_hostname: "{{ openshift_master_cluster_hostname | default(None) }}"
+      cluster_public_hostname: "{{ openshift_master_cluster_public_hostname | default(None) }}"
+      cluster_defer_ha: "{{ openshift_master_cluster_defer_ha | default(None) }}"
       debug_level: "{{ openshift_master_debug_level | default(openshift.common.debug_level) }}"
       api_port: "{{ openshift_master_api_port | default(None) }}"
       api_url: "{{ openshift_master_api_url | default(None) }}"
@@ -114,12 +121,26 @@
 
 - name: Start and enable openshift-master
   service: name=openshift-master enabled=yes state=started
+  when: not openshift_master_ha
   register: start_result
 
 - name: pause to prevent service restart from interfering with bootstrapping
   pause: seconds=30
   when: start_result | changed
 
+- name: Install cluster packagese
+  yum: pkg=pcs state=present
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+  register: install_result
+
+- name: Start and enable cluster service
+  service: name=pcsd enabled=yes state=started
+  when: openshift_master_ha and not openshift.master.cluster_defer_ha | bool
+
+- name: Set the cluster user password
+  shell: echo {{ openshift_master_cluster_password | quote }} | passwd --stdin hacluster
+  when: install_result | changed
+
 - name: Create the OpenShift client config dir(s)
   file:
     path: "~{{ item }}/.kube"
diff --git a/roles/openshift_master_ca/tasks/main.yml b/roles/openshift_master_ca/tasks/main.yml
index 8163ecd7f..03eb7e15f 100644
--- a/roles/openshift_master_ca/tasks/main.yml
+++ b/roles/openshift_master_ca/tasks/main.yml
@@ -14,7 +14,7 @@
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ openshift.common.hostname }},{{ openshift.common.public_hostname }}
+      --hostnames={{ openshift.common.all_hostnames | join(',') }}
       --master={{ openshift.master.api_url }}
       --public-master={{ openshift.master.public_api_url }}
       --cert-dir={{ openshift_master_config_dir }} --overwrite=false
diff --git a/roles/openshift_master_certificates/tasks/main.yml b/roles/openshift_master_certificates/tasks/main.yml
index b5a3f8e40..297d53bcd 100644
--- a/roles/openshift_master_certificates/tasks/main.yml
+++ b/roles/openshift_master_certificates/tasks/main.yml
@@ -7,14 +7,20 @@
   with_items: masters_needing_certs
 
 - file:
-    src: "{{ openshift_master_ca_cert }}"
-    dest: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/ca.crt"
-  with_items: masters_needing_certs
+    src: "{{ openshift_master_config_dir }}/{{ item.1 }}"
+    dest: "{{ openshift_generated_configs_dir }}/{{ item.0.master_cert_subdir }}/{{ item.1 }}"
+    state: hard
+  with_nested:
+  - masters_needing_certs
+  - - ca.crt
+    - ca.key
+    - ca.serial.txt
+
 
 - name: Create the master certificates if they do not already exist
   command: >
     {{ openshift.common.admin_binary }} create-master-certs
-      --hostnames={{ item.openshift.common.hostname }},{{ item.openshift.common.public_hostname }}
+      --hostnames={{ item.openshift.common.all_hostnames | join(',') }}
       --master={{ item.openshift.master.api_url }}
       --public-master={{ item.openshift.master.public_api_url }}
       --cert-dir={{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}
@@ -22,3 +28,5 @@
   args:
     creates: "{{ openshift_generated_configs_dir }}/{{ item.master_cert_subdir }}/master.server.crt"
   with_items: masters_needing_certs
+
+
diff --git a/roles/openshift_master_certificates/vars/main.yml b/roles/openshift_master_certificates/vars/main.yml
index 6e577b13b..6214f7918 100644
--- a/roles/openshift_master_certificates/vars/main.yml
+++ b/roles/openshift_master_certificates/vars/main.yml
@@ -1,6 +1,3 @@
 ---
 openshift_generated_configs_dir: /etc/openshift/generated-configs
 openshift_master_config_dir: /etc/openshift/master
-openshift_master_ca_cert: "{{ openshift_master_config_dir }}/ca.crt"
-openshift_master_ca_key: "{{ openshift_master_config_dir }}/ca.key"
-openshift_master_ca_serial: "{{ openshift_master_config_dir }}/ca.serial.txt"
diff --git a/roles/openshift_master_cluster/README.md b/roles/openshift_master_cluster/README.md
new file mode 100644
index 000000000..f150981fa
--- /dev/null
+++ b/roles/openshift_master_cluster/README.md
@@ -0,0 +1,34 @@
+OpenShift Master Cluster
+========================
+
+TODO
+
+Requirements
+------------
+
+TODO
+
+Role Variables
+--------------
+
+TODO
+
+Dependencies
+------------
+
+TODO
+
+Example Playbook
+----------------
+
+TODO
+
+License
+-------
+
+Apache License Version 2.0
+
+Author Information
+------------------
+
+Jason DeTiberus (jdetiber@redhat.com)
diff --git a/roles/openshift_master_cluster/meta/main.yml b/roles/openshift_master_cluster/meta/main.yml
new file mode 100644
index 000000000..f3236e850
--- /dev/null
+++ b/roles/openshift_master_cluster/meta/main.yml
@@ -0,0 +1,16 @@
+---
+galaxy_info:
+  author: Jason DeTiberus
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- { role: openshift_facts }
diff --git a/roles/openshift_master_cluster/tasks/configure.yml b/roles/openshift_master_cluster/tasks/configure.yml
new file mode 100644
index 000000000..9a2546bca
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/configure.yml
@@ -0,0 +1,43 @@
+---
+- fail:
+    msg: This role requires that openshift_master_cluster_vip is set
+  when: openshift_master_cluster_vip is not defined or not openshift_master_cluster_vip
+- fail:
+    msg: This role requires that openshift_master_cluster_public_vip is set
+  when: openshift_master_cluster_public_vip is not defined or not openshift_master_cluster_public_vip
+
+- name: Authenticate to the cluster
+  command: pcs cluster auth -u hacluster -p {{ openshift_master_cluster_password }} {{ omc_cluster_hosts }}
+
+- name: Create the cluster
+  command: pcs cluster setup --name openshift_master {{ omc_cluster_hosts }}
+
+- name: Start the cluster
+  command: pcs cluster start --all
+
+- name: Enable the cluster on all nodes
+  command: pcs cluster enable --all
+
+- name: Set default resource stickiness
+  command: pcs resource defaults resource-stickiness=100
+
+- name: Add the cluster VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_vip }} --group openshift-master
+
+- name: Add the cluster public VIP resource
+  command: pcs resource create virtual-ip IPaddr2 ip={{ openshift_master_cluster_public_vip }} --group openshift-master
+  when: openshift_master_cluster_public_vip != openshift_master_cluster_vip
+
+- name: Add the cluster openshift-master service resource
+  command: pcs resource create master systemd:openshift-master --group openshift-master
+
+- name: Disable stonith
+  command: pcs property set stonith-enabled=false
+
+# TODO: handle case where api port is not 8443
+- name: Wait for the clustered master service to be available
+  wait_for:
+    host: "{{ openshift_master_cluster_vip }}"
+    port: 8443
+    state: started
+    timeout: 300
diff --git a/roles/openshift_master_cluster/tasks/configure_deferred.yml b/roles/openshift_master_cluster/tasks/configure_deferred.yml
new file mode 100644
index 000000000..a80b6c5b4
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/configure_deferred.yml
@@ -0,0 +1,8 @@
+---
+- debug: msg="Deferring config"
+
+- name: Start and enable openshift-master
+  service:
+    name: openshift-master
+    state: started
+    enabled: yes
diff --git a/roles/openshift_master_cluster/tasks/main.yml b/roles/openshift_master_cluster/tasks/main.yml
new file mode 100644
index 000000000..315947183
--- /dev/null
+++ b/roles/openshift_master_cluster/tasks/main.yml
@@ -0,0 +1,13 @@
+---
+- name: Test if cluster is already configured
+  command: pcs status
+  register: pcs_status
+  changed_when: false
+  failed_when: false
+  when: not openshift.master.cluster_defer_ha | bool
+
+- include: configure.yml
+  when: "pcs_status | failed and 'Error: cluster is not currently running on this node' in pcs_status.stderr"
+
+- include: configure_deferred.yml
+  when: openshift.master.cluster_defer_ha | bool
diff --git a/roles/openshift_node_certificates/tasks/main.yml b/roles/openshift_node_certificates/tasks/main.yml
index 64a799dfb..c9f02aaf0 100644
--- a/roles/openshift_node_certificates/tasks/main.yml
+++ b/roles/openshift_node_certificates/tasks/main.yml
@@ -25,7 +25,7 @@
   command: >
     {{ openshift.common.admin_binary }} create-server-cert
       --cert=server.crt --key=server.key --overwrite=true
-      --hostnames={{ [item.openshift.common.hostname, item.openshift.common.public_hostname]|unique|join(",") }}
+      --hostnames={{ openshift.common.all_hostnames |join(",") }}
       --signer-cert={{ openshift_master_ca_cert }}
       --signer-key={{ openshift_master_ca_key }}
       --signer-serial={{ openshift_master_ca_serial }}