From 57abf26c30e306d1527e92a61a94088e542bb006 Mon Sep 17 00:00:00 2001 From: Andrew Butcher Date: Mon, 16 May 2016 11:45:32 -0400 Subject: Separate master and haproxy config playbooks. * Move haproxy configuration to a separate openshift-loadbalancer play. * Move the haproxy role to openshift_loadbalancer. * Add openshift_loadbalancer* facts which drive haproxy configuration. --- filter_plugins/oo_filters.py | 5 +- playbooks/common/openshift-cluster/config.yml | 2 + playbooks/common/openshift-loadbalancer/config.yml | 5 ++ .../common/openshift-loadbalancer/filter_plugins | 1 + .../common/openshift-loadbalancer/lookup_plugins | 1 + playbooks/common/openshift-loadbalancer/roles | 1 + .../common/openshift-loadbalancer/service.yml | 20 ++++++ playbooks/common/openshift-master/config.yml | 27 -------- roles/haproxy/README.md | 34 ---------- roles/haproxy/defaults/main.yml | 22 ------- roles/haproxy/handlers/main.yml | 6 -- roles/haproxy/meta/main.yml | 14 ---- roles/haproxy/tasks/main.yml | 43 ------------ roles/haproxy/templates/haproxy.cfg.j2 | 76 ---------------------- roles/openshift_facts/library/openshift_facts.py | 8 +++ roles/openshift_loadbalancer/README.md | 34 ++++++++++ roles/openshift_loadbalancer/defaults/main.yml | 22 +++++++ roles/openshift_loadbalancer/handlers/main.yml | 6 ++ roles/openshift_loadbalancer/meta/main.yml | 15 +++++ roles/openshift_loadbalancer/tasks/main.yml | 73 +++++++++++++++++++++ .../templates/haproxy.cfg.j2 | 76 ++++++++++++++++++++++ 21 files changed, 266 insertions(+), 225 deletions(-) create mode 100644 playbooks/common/openshift-loadbalancer/config.yml create mode 120000 playbooks/common/openshift-loadbalancer/filter_plugins create mode 120000 playbooks/common/openshift-loadbalancer/lookup_plugins create mode 120000 playbooks/common/openshift-loadbalancer/roles create mode 100644 playbooks/common/openshift-loadbalancer/service.yml delete mode 100644 roles/haproxy/README.md delete mode 100644 roles/haproxy/defaults/main.yml delete mode 100644 roles/haproxy/handlers/main.yml delete mode 100644 roles/haproxy/meta/main.yml delete mode 100644 roles/haproxy/tasks/main.yml delete mode 100644 roles/haproxy/templates/haproxy.cfg.j2 create mode 100644 roles/openshift_loadbalancer/README.md create mode 100644 roles/openshift_loadbalancer/defaults/main.yml create mode 100644 roles/openshift_loadbalancer/handlers/main.yml create mode 100644 roles/openshift_loadbalancer/meta/main.yml create mode 100644 roles/openshift_loadbalancer/tasks/main.yml create mode 100644 roles/openshift_loadbalancer/templates/haproxy.cfg.j2 diff --git a/filter_plugins/oo_filters.py b/filter_plugins/oo_filters.py index e7409bf22..b81c3bf7f 100644 --- a/filter_plugins/oo_filters.py +++ b/filter_plugins/oo_filters.py @@ -306,7 +306,7 @@ class FilterModule(object): return string.split(separator) @staticmethod - def oo_haproxy_backend_masters(hosts): + def oo_haproxy_backend_masters(hosts, port): """ This takes an array of dicts and returns an array of dicts to be used as a backend for the haproxy role """ @@ -314,8 +314,7 @@ class FilterModule(object): for idx, host_info in enumerate(hosts): server = dict(name="master%s" % idx) server_ip = host_info['openshift']['common']['ip'] - server_port = host_info['openshift']['master']['api_port'] - server['address'] = "%s:%s" % (server_ip, server_port) + server['address'] = "%s:%s" % (server_ip, port) server['opts'] = 'check' servers.append(server) return servers diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml index 903babc45..5fec11541 100644 --- a/playbooks/common/openshift-cluster/config.yml +++ b/playbooks/common/openshift-cluster/config.yml @@ -31,6 +31,8 @@ - include: ../openshift-nfs/config.yml +- include: ../openshift-loadbalancer/config.yml + - include: ../openshift-master/config.yml - include: additional_config.yml diff --git a/playbooks/common/openshift-loadbalancer/config.yml b/playbooks/common/openshift-loadbalancer/config.yml new file mode 100644 index 000000000..f4392173a --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/config.yml @@ -0,0 +1,5 @@ +--- +- name: Configure load balancers + hosts: oo_lb_to_config + roles: + - role: openshift_loadbalancer diff --git a/playbooks/common/openshift-loadbalancer/filter_plugins b/playbooks/common/openshift-loadbalancer/filter_plugins new file mode 120000 index 000000000..99a95e4ca --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/filter_plugins @@ -0,0 +1 @@ +../../../filter_plugins \ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/lookup_plugins b/playbooks/common/openshift-loadbalancer/lookup_plugins new file mode 120000 index 000000000..ac79701db --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/lookup_plugins @@ -0,0 +1 @@ +../../../lookup_plugins \ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/roles b/playbooks/common/openshift-loadbalancer/roles new file mode 120000 index 000000000..e2b799b9d --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/roles @@ -0,0 +1 @@ +../../../roles/ \ No newline at end of file diff --git a/playbooks/common/openshift-loadbalancer/service.yml b/playbooks/common/openshift-loadbalancer/service.yml new file mode 100644 index 000000000..19fffd5e9 --- /dev/null +++ b/playbooks/common/openshift-loadbalancer/service.yml @@ -0,0 +1,20 @@ +--- +- name: Populate g_service_nodes host group if needed + hosts: localhost + connection: local + become: no + gather_facts: no + tasks: + - fail: msg="new_cluster_state is required to be injected in this playbook" + when: new_cluster_state is not defined + + - name: Evaluate g_service_lb + add_host: name={{ item }} groups=g_service_lb + with_items: oo_host_group_exp | default([]) + +- name: Change state on lb instance(s) + hosts: g_service_lb + connection: ssh + gather_facts: no + tasks: + - service: name=hapoxy state="{{ new_cluster_state }}" diff --git a/playbooks/common/openshift-master/config.yml b/playbooks/common/openshift-master/config.yml index fe0e2e666..e9337270b 100644 --- a/playbooks/common/openshift-master/config.yml +++ b/playbooks/common/openshift-master/config.yml @@ -156,33 +156,6 @@ - master.etcd-ca.crt when: etcd_client_certs_missing is defined and etcd_client_certs_missing -- name: Configure load balancers - hosts: oo_lb_to_config - vars: - sync_tmpdir: "{{ hostvars.localhost.g_master_mktemp.stdout }}" - haproxy_limit_nofile: 100000 - haproxy_global_maxconn: 20000 - haproxy_default_maxconn: 20000 - haproxy_frontend_port: "{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}" - haproxy_frontends: - - name: atomic-openshift-api - mode: tcp - options: - - tcplog - binds: - - "*:{{ hostvars[groups.oo_first_master.0].openshift.master.api_port }}" - default_backend: atomic-openshift-api - haproxy_backends: - - name: atomic-openshift-api - mode: tcp - option: tcplog - balance: source - servers: "{{ hostvars | oo_select_keys(groups['oo_masters']) | oo_haproxy_backend_masters }}" - roles: - - role: openshift_facts - - role: haproxy - when: hostvars[groups.oo_first_master.0].openshift.master.ha | bool - - name: Check for cached session secrets hosts: oo_first_master roles: diff --git a/roles/haproxy/README.md b/roles/haproxy/README.md deleted file mode 100644 index 5bc415066..000000000 --- a/roles/haproxy/README.md +++ /dev/null @@ -1,34 +0,0 @@ -HAProxy -======= - -TODO - -Requirements ------------- - -TODO - -Role Variables --------------- - -TODO - -Dependencies ------------- - -TODO - -Example Playbook ----------------- - -TODO - -License -------- - -Apache License, Version 2.0 - -Author Information ------------------- - -Jason DeTiberus (jdetiber@redhat.com) diff --git a/roles/haproxy/defaults/main.yml b/roles/haproxy/defaults/main.yml deleted file mode 100644 index a1524cfe1..000000000 --- a/roles/haproxy/defaults/main.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -haproxy_frontend_port: 80 - -haproxy_frontends: -- name: main - binds: - - "*:80" - default_backend: default - -haproxy_backends: -- name: default - balance: roundrobin - servers: - - name: web01 - address: 127.0.0.1:9000 - opts: check - -os_firewall_allow: -- service: haproxy stats - port: "9000/tcp" -- service: haproxy balance - port: "{{ haproxy_frontend_port }}/tcp" diff --git a/roles/haproxy/handlers/main.yml b/roles/haproxy/handlers/main.yml deleted file mode 100644 index 5b8691b26..000000000 --- a/roles/haproxy/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: restart haproxy - service: - name: haproxy - state: restarted - when: not (haproxy_start_result_changed | default(false) | bool) diff --git a/roles/haproxy/meta/main.yml b/roles/haproxy/meta/main.yml deleted file mode 100644 index 0fad106a9..000000000 --- a/roles/haproxy/meta/main.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -galaxy_info: - author: Jason DeTiberus - description: HAProxy - company: Red Hat, Inc. - license: Apache License, Version 2.0 - min_ansible_version: 1.9 - platforms: - - name: EL - versions: - - 7 -dependencies: -- { role: os_firewall } -- { role: openshift_repos } diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml deleted file mode 100644 index 837fa67db..000000000 --- a/roles/haproxy/tasks/main.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Install haproxy - action: "{{ ansible_pkg_mgr }} name=haproxy state=present" - when: not openshift.common.is_containerized | bool - -- name: Configure systemd service directory for haproxy - file: - path: /etc/systemd/system/haproxy.service.d - state: directory - when: haproxy_limit_nofile is defined - -- name: Configure the nofile limits for haproxy - ini_file: - dest: /etc/systemd/system/haproxy.service.d/limits.conf - section: Service - option: LimitNOFILE - value: "{{ haproxy_limit_nofile }}" - when: haproxy_limit_nofile is defined - notify: restart haproxy - register: nofile_limit_result - -- name: Reload systemd if needed - command: systemctl daemon-reload - when: nofile_limit_result | changed - -- name: Configure haproxy - template: - src: haproxy.cfg.j2 - dest: /etc/haproxy/haproxy.cfg - owner: root - group: root - mode: 0644 - notify: restart haproxy - -- name: Enable and start haproxy - service: - name: haproxy - state: started - enabled: yes - register: start_result - -- set_fact: - haproxy_start_result_changed: "{{ start_result | changed }}" diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2 deleted file mode 100644 index cb4380971..000000000 --- a/roles/haproxy/templates/haproxy.cfg.j2 +++ /dev/null @@ -1,76 +0,0 @@ -# Global settings -#--------------------------------------------------------------------- -global - chroot /var/lib/haproxy - pidfile /var/run/haproxy.pid - maxconn {{ haproxy_global_maxconn | default('4000') }} - user haproxy - group haproxy - daemon - - # turn on stats unix socket - stats socket /var/lib/haproxy/stats - -#--------------------------------------------------------------------- -# common defaults that all the 'listen' and 'backend' sections will -# use if not designated in their block -#--------------------------------------------------------------------- -defaults - mode http - log global - option httplog - option dontlognull -# option http-server-close - option forwardfor except 127.0.0.0/8 - option redispatch - retries 3 - timeout http-request 10s - timeout queue 1m - timeout connect 10s - timeout client 300s - timeout server 300s - timeout http-keep-alive 10s - timeout check 10s - maxconn {{ haproxy_default_maxconn | default('3000') }} - -listen stats :9000 - mode http - stats enable - stats uri / - -{% for frontend in haproxy_frontends %} -frontend {{ frontend.name }} -{% for bind in frontend.binds %} - bind {{ bind }} -{% endfor %} - default_backend {{ frontend.default_backend }} -{% if 'mode' in frontend %} - mode {{ frontend.mode }} -{% endif %} -{% if 'options' in frontend %} -{% for option in frontend.options %} - option {{ option }} -{% endfor %} -{% endif %} -{% if 'redirects' in frontend %} -{% for redirect in frontend.redirects %} - redirect {{ redirect }} -{% endfor %} -{% endif %} -{% endfor %} - -{% for backend in haproxy_backends %} -backend {{ backend.name }} - balance {{ backend.balance }} -{% if 'mode' in backend %} - mode {{ backend.mode }} -{% endif %} -{% if 'options' in backend %} -{% for option in backend.options %} - option {{ option }} -{% endfor %} -{% endif %} -{% for server in backend.servers %} - server {{ server.name }} {{ server.address }} {{ server.opts }} -{% endfor %} -{% endfor %} diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index b13343776..11203de17 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -1554,6 +1554,7 @@ class OpenShiftFacts(object): 'docker', 'etcd', 'hosted', + 'loadbalancer', 'master', 'node'] @@ -1763,6 +1764,13 @@ class OpenShiftFacts(object): router=dict() ) + if 'loadbalancer' in roles: + loadbalancer = dict(frontend_port='8443', + default_maxconn='20000', + global_maxconn='20000', + limit_nofile='100000') + defaults['loadbalancer'] = loadbalancer + return defaults def guess_host_provider(self): diff --git a/roles/openshift_loadbalancer/README.md b/roles/openshift_loadbalancer/README.md new file mode 100644 index 000000000..81fc282be --- /dev/null +++ b/roles/openshift_loadbalancer/README.md @@ -0,0 +1,34 @@ +OpenShift HAProxy Loadbalancer +============================== + +TODO + +Requirements +------------ + +TODO + +Role Variables +-------------- + +TODO + +Dependencies +------------ + +TODO + +Example Playbook +---------------- + +TODO + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +Jason DeTiberus (jdetiber@redhat.com) diff --git a/roles/openshift_loadbalancer/defaults/main.yml b/roles/openshift_loadbalancer/defaults/main.yml new file mode 100644 index 000000000..a1524cfe1 --- /dev/null +++ b/roles/openshift_loadbalancer/defaults/main.yml @@ -0,0 +1,22 @@ +--- +haproxy_frontend_port: 80 + +haproxy_frontends: +- name: main + binds: + - "*:80" + default_backend: default + +haproxy_backends: +- name: default + balance: roundrobin + servers: + - name: web01 + address: 127.0.0.1:9000 + opts: check + +os_firewall_allow: +- service: haproxy stats + port: "9000/tcp" +- service: haproxy balance + port: "{{ haproxy_frontend_port }}/tcp" diff --git a/roles/openshift_loadbalancer/handlers/main.yml b/roles/openshift_loadbalancer/handlers/main.yml new file mode 100644 index 000000000..5b8691b26 --- /dev/null +++ b/roles/openshift_loadbalancer/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: restart haproxy + service: + name: haproxy + state: restarted + when: not (haproxy_start_result_changed | default(false) | bool) diff --git a/roles/openshift_loadbalancer/meta/main.yml b/roles/openshift_loadbalancer/meta/main.yml new file mode 100644 index 000000000..fe336acf7 --- /dev/null +++ b/roles/openshift_loadbalancer/meta/main.yml @@ -0,0 +1,15 @@ +--- +galaxy_info: + author: Jason DeTiberus + description: OpenShift haproxy loadbalancer + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 1.9 + platforms: + - name: EL + versions: + - 7 +dependencies: +- role: openshift_facts +- role: os_firewall +- role: openshift_repos diff --git a/roles/openshift_loadbalancer/tasks/main.yml b/roles/openshift_loadbalancer/tasks/main.yml new file mode 100644 index 000000000..5514aa70b --- /dev/null +++ b/roles/openshift_loadbalancer/tasks/main.yml @@ -0,0 +1,73 @@ +--- +- name: Set haproxy frontend port + openshift_facts: + role: loadbalancer + local_facts: + frontend_port: "{{ openshift_master_api_port | default(None) }}" + +- name: Set loadbalancer facts + openshift_facts: + role: loadbalancer + local_facts: + limit_nofile: "{{ openshift_loadbalancer_limit_nofile | default(None) }}" + default_maxconn: "{{ openshift_loadbalancer_default_maxconn | default(None) }}" + global_maxconn: "{{ openshift_loadbalancer_global_maxconn | default(None) }}" + frontends: + - name: atomic-openshift-api + mode: tcp + options: + - tcplog + binds: + - "*:{{ openshift.loadbalancer.frontend_port }}" + default_backend: atomic-openshift-api + backends: + - name: atomic-openshift-api + mode: tcp + option: tcplog + balance: source + servers: "{{ hostvars + | oo_select_keys(groups['oo_masters']) + | oo_haproxy_backend_masters(openshift.loadbalancer.frontend_port) }}" + +- name: Install haproxy + action: "{{ ansible_pkg_mgr }} name=haproxy state=present" + when: not openshift.common.is_containerized | bool + +- name: Configure systemd service directory for haproxy + file: + path: /etc/systemd/system/haproxy.service.d + state: directory + when: "'limit_nofile' in openshift.loadbalancer" + +- name: Configure the nofile limits for haproxy + ini_file: + dest: /etc/systemd/system/haproxy.service.d/limits.conf + section: Service + option: LimitNOFILE + value: "{{ openshift.loadbalancer.limit_nofile }}" + when: "'limit_nofile' in openshift.loadbalancer" + notify: restart haproxy + register: nofile_limit_result + +- name: Reload systemd if needed + command: systemctl daemon-reload + when: nofile_limit_result | changed + +- name: Configure haproxy + template: + src: haproxy.cfg.j2 + dest: /etc/haproxy/haproxy.cfg + owner: root + group: root + mode: 0644 + notify: restart haproxy + +- name: Enable and start haproxy + service: + name: haproxy + state: started + enabled: yes + register: start_result + +- set_fact: + haproxy_start_result_changed: "{{ start_result | changed }}" diff --git a/roles/openshift_loadbalancer/templates/haproxy.cfg.j2 b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2 new file mode 100644 index 000000000..05e360d3b --- /dev/null +++ b/roles/openshift_loadbalancer/templates/haproxy.cfg.j2 @@ -0,0 +1,76 @@ +# Global settings +#--------------------------------------------------------------------- +global + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn {{ openshift.loadbalancer.global_maxconn }} + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +#--------------------------------------------------------------------- +# common defaults that all the 'listen' and 'backend' sections will +# use if not designated in their block +#--------------------------------------------------------------------- +defaults + mode http + log global + option httplog + option dontlognull +# option http-server-close + option forwardfor except 127.0.0.0/8 + option redispatch + retries 3 + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 300s + timeout server 300s + timeout http-keep-alive 10s + timeout check 10s + maxconn {{ openshift.loadbalancer.default_maxconn }} + +listen stats :9000 + mode http + stats enable + stats uri / + +{% for frontend in openshift.loadbalancer.frontends %} +frontend {{ frontend.name }} +{% for bind in frontend.binds %} + bind {{ bind }} +{% endfor %} + default_backend {{ frontend.default_backend }} +{% if 'mode' in frontend %} + mode {{ frontend.mode }} +{% endif %} +{% if 'options' in frontend %} +{% for option in frontend.options %} + option {{ option }} +{% endfor %} +{% endif %} +{% if 'redirects' in frontend %} +{% for redirect in frontend.redirects %} + redirect {{ redirect }} +{% endfor %} +{% endif %} +{% endfor %} + +{% for backend in openshift.loadbalancer.backends %} +backend {{ backend.name }} + balance {{ backend.balance }} +{% if 'mode' in backend %} + mode {{ backend.mode }} +{% endif %} +{% if 'options' in backend %} +{% for option in backend.options %} + option {{ option }} +{% endfor %} +{% endif %} +{% for server in backend.servers %} + server {{ server.name }} {{ server.address }} {{ server.opts }} +{% endfor %} +{% endfor %} -- cgit v1.2.1