diff options
Diffstat (limited to 'roles/openshift_logging_elasticsearch')
9 files changed, 248 insertions, 24 deletions
diff --git a/roles/openshift_logging_elasticsearch/handlers/main.yml b/roles/openshift_logging_elasticsearch/handlers/main.yml new file mode 100644 index 000000000..fa56897d0 --- /dev/null +++ b/roles/openshift_logging_elasticsearch/handlers/main.yml @@ -0,0 +1,13 @@ +--- +- name: "Restarting logging-{{ _cluster_component }} cluster" + listen: "restart elasticsearch" + include_tasks: restart_cluster.yml + with_items: "{{ _restart_logging_components }}" + loop_control: + loop_var: _cluster_component + when: not logging_elasticsearch_rollout_override | bool + +## Stop this from running more than once +- set_fact: + logging_elasticsearch_rollout_override: True + listen: "restart elasticsearch" diff --git a/roles/openshift_logging_elasticsearch/meta/main.yaml b/roles/openshift_logging_elasticsearch/meta/main.yaml index 6a9a6539c..e93d6b73e 100644 --- a/roles/openshift_logging_elasticsearch/meta/main.yaml +++ b/roles/openshift_logging_elasticsearch/meta/main.yaml @@ -14,3 +14,4 @@ galaxy_info: dependencies: - role: lib_openshift - role: openshift_facts +- role: lib_utils diff --git a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml index c53a06019..c55e7c5ea 100644 --- a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml @@ -15,3 +15,5 @@ - fail: msg: Invalid version specified for Elasticsearch when: es_version not in __allowed_es_versions + +- include_tasks: get_es_version.yml diff --git a/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml new file mode 100644 index 000000000..9182bddb2 --- /dev/null +++ b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml @@ -0,0 +1,42 @@ +--- +- command: > + oc get pod -l component=es,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _cluster_pods + +- name: "Getting ES version for logging-es cluster" + command: > + oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/' + register: _curl_output + when: _cluster_pods.stdout_lines | count > 0 + +- command: > + oc get pod -l component=es-ops,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _ops_cluster_pods + +- name: "Getting ES version for logging-es-ops cluster" + command: > + oc exec {{ _ops_cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/' + register: _ops_curl_output + when: _ops_cluster_pods.stdout_lines | count > 0 + +- set_fact: + _es_output: "{{ _curl_output.stdout | from_json }}" + when: _curl_output.stdout is defined + +- set_fact: + _es_ops_output: "{{ _ops_curl_output.stdout | from_json }}" + when: _ops_curl_output.stdout is defined + +- set_fact: + _es_installed_version: "{{ _es_output.version.number }}" + when: + - _es_output is defined + - _es_output.version is defined + - _es_output.version.number is defined + +- set_fact: + _es_ops_installed_version: "{{ _es_ops_output.version.number }}" + when: + - _es_ops_output is defined + - _es_ops_output.version is defined + - _es_ops_output.version.number is defined diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml index 8f2050043..ff5ad1045 100644 --- a/roles/openshift_logging_elasticsearch/tasks/main.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml @@ -15,10 +15,10 @@ elasticsearch_name: "{{ 'logging-elasticsearch' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '')) }}" es_component: "{{ 'es' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '') ) }}" -- name: Set default image variables based on deployment_type +- name: Set default image variables based on openshift_deployment_type include_vars: "{{ var_file_name }}" with_first_found: - - "{{ openshift_deployment_type | default(deployment_type) }}.yml" + - "{{ openshift_deployment_type }}.yml" - "default_images.yml" loop_control: loop_var: var_file_name @@ -32,6 +32,18 @@ - include_tasks: determine_version.yaml +- set_fact: + full_restart_cluster: True + when: + - _es_installed_version is defined + - _es_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int + +- set_fact: + full_restart_cluster: True + when: + - _es_ops_installed_version is defined + - _es_ops_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int + # allow passing in a tempdir - name: Create temp directory for doing work in command: mktemp -d /tmp/openshift-logging-ansible-XXXXXX @@ -111,7 +123,7 @@ - name: Create logging-metrics-reader-role command: > - {{ openshift.common.client_binary }} + {{ openshift_client_binary }} --config={{ openshift.common.config_base }}/master/admin.kubeconfig -n "{{ openshift_logging_elasticsearch_namespace }}" create -f "{{mktemp.stdout}}/templates/logging-metrics-role.yml" @@ -168,33 +180,33 @@ when: es_logging_contents is undefined changed_when: no -- set_fact: - __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}" - __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}" - - template: src: elasticsearch.yml.j2 dest: "{{ tempdir }}/elasticsearch.yml" vars: allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}" - es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}" - es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}" + es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}" + es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}" es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}" when: es_config_contents is undefined changed_when: no -- copy: - content: "{{ es_logging_contents }}" - dest: "{{ tempdir }}/elasticsearch-logging.yml" - when: es_logging_contents is defined - changed_when: no - -- copy: - content: "{{ es_config_contents }}" - dest: "{{ tempdir }}/elasticsearch.yml" - when: es_config_contents is defined - changed_when: no +# create diff between current configmap files and our current files +# NOTE: include_role must be used instead of import_role because +# this task file is looped over from another role. +- include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-elasticsearch" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "elasticsearch.yml" + new_file: "{{ tempdir }}/elasticsearch.yml" + protected_lines: ["number_of_shards", "number_of_replicas"] + - current_file: "logging.yml" + new_file: "{{ tempdir }}/elasticsearch-logging.yml" - name: Set ES configmap oc_configmap: @@ -204,7 +216,21 @@ from_file: elasticsearch.yml: "{{ tempdir }}/elasticsearch.yml" logging.yml: "{{ tempdir }}/elasticsearch-logging.yml" + register: es_config_creation + notify: "restart elasticsearch" +- when: es_config_creation.changed | bool + block: + - set_fact: + _restart_logging_components: "{{ _restart_logging_components | default([]) + [es_component] | unique }}" + + - shell: > + oc get dc -l component="{{ es_component }}" -n "{{ openshift_logging_elasticsearch_namespace }}" -o name | cut -d'/' -f2 + register: _es_dcs + + - set_fact: + _restart_logging_nodes: "{{ _restart_logging_nodes | default([]) + [_es_dcs.stdout] | unique }}" + when: _es_dcs.stdout != "" # secret - name: Set ES secret @@ -338,7 +364,7 @@ delete_after: true - set_fact: - es_deploy_name: "logging-{{ es_component }}-{{ openshift_logging_elasticsearch_deployment_type }}-{{ 8 | oo_random_word('abcdefghijklmnopqrstuvwxyz0123456789') }}" + es_deploy_name: "logging-{{ es_component }}-{{ openshift_logging_elasticsearch_deployment_type }}-{{ 8 | lib_utils_oo_random_word('abcdefghijklmnopqrstuvwxyz0123456789') }}" when: openshift_logging_elasticsearch_deployment_name == "" - set_fact: @@ -375,6 +401,13 @@ files: - "{{ tempdir }}/templates/logging-es-dc.yml" delete_after: true + register: es_dc_creation + notify: "restart elasticsearch" + +- set_fact: + _restart_logging_components: "{{ _restart_logging_components | default([]) + [es_component] | unique }}" + _restart_logging_nodes: "{{ _restart_logging_nodes | default([]) + [es_deploy_name] | unique }}" + when: es_dc_creation.changed | bool - name: Retrieving the cert to use when generating secrets for the {{ es_component }} component slurp: diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml new file mode 100644 index 000000000..d55beec86 --- /dev/null +++ b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml @@ -0,0 +1,91 @@ +--- +# Disable external communication for {{ _cluster_component }} +- name: Disable external communication for logging-{{ _cluster_component }} + oc_service: + state: present + name: "logging-{{ _cluster_component }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + selector: + component: "{{ _cluster_component }}" + provider: openshift + connection: blocked + labels: + logging-infra: 'support' + ports: + - port: 9200 + targetPort: "restapi" + when: + - full_restart_cluster | bool + +## get all pods for the cluster +- command: > + oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _cluster_pods + +- name: "Disable shard balancing for logging-{{ _cluster_component }} cluster" + command: > + oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "none" } }' + register: _disable_output + changed_when: "'\"acknowledged\":true' in _disable_output.stdout" + when: _cluster_pods.stdout_lines | count > 0 + +# Flush ES +- name: "Flushing for logging-{{ _cluster_component }} cluster" + command: > + oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_flush/synced' + register: _flush_output + changed_when: "'\"acknowledged\":true' in _flush_output.stdout" + when: + - _cluster_pods.stdout_lines | count > 0 + - full_restart_cluster | bool + +- command: > + oc get dc -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _cluster_dcs + +## restart all dcs for full restart +- name: "Restart ES node {{ _es_node }}" + include_tasks: restart_es_node.yml + with_items: "{{ _cluster_dcs }}" + loop_control: + loop_var: _es_node + when: + - full_restart_cluster | bool + +## restart the node if it's dc is in the list of nodes to restart? +- name: "Restart ES node {{ _es_node }}" + include_tasks: restart_es_node.yml + with_items: "{{ _restart_logging_nodes }}" + loop_control: + loop_var: _es_node + when: + - not full_restart_cluster | bool + - _es_node in _cluster_dcs.stdout + +## we may need a new first pod to run against -- fetch them all again +- command: > + oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _cluster_pods + +- name: "Enable shard balancing for logging-{{ _cluster_component }} cluster" + command: > + oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "all" } }' + register: _enable_output + changed_when: "'\"acknowledged\":true' in _enable_output.stdout" + +# Reenable external communication for {{ _cluster_component }} +- name: Reenable external communication for logging-{{ _cluster_component }} + oc_service: + state: present + name: "logging-{{ _cluster_component }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + selector: + component: "{{ _cluster_component }}" + provider: openshift + labels: + logging-infra: 'support' + ports: + - port: 9200 + targetPort: "restapi" + when: + - full_restart_cluster | bool diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml new file mode 100644 index 000000000..6d0df40c8 --- /dev/null +++ b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml @@ -0,0 +1,37 @@ +--- +- name: "Rolling out new pod(s) for {{ _es_node }}" + command: > + oc rollout latest {{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} + +- name: "Waiting for {{ _es_node }} to finish scaling up" + oc_obj: + state: list + name: "{{ _es_node }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + kind: dc + register: _dc_output + until: + - _dc_output.results.results[0].status is defined + - _dc_output.results.results[0].status.readyReplicas is defined + - _dc_output.results.results[0].status.readyReplicas > 0 + - _dc_output.results.results[0].status.updatedReplicas is defined + - _dc_output.results.results[0].status.updatedReplicas > 0 + retries: 60 + delay: 30 + +- name: Gettings name(s) of replica pod(s) + command: > + oc get pods -l deploymentconfig={{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _pods + +- name: "Waiting for ES to be ready for {{ _es_node }}" + shell: > + oc exec "{{ _pod }}" -c elasticsearch -n "{{ openshift_logging_elasticsearch_namespace }}" -- {{ __es_local_curl }} https://localhost:9200/_cat/health | cut -d' ' -f4 + with_items: "{{ _pods.stdout.split(' ') }}" + loop_control: + loop_var: _pod + register: _pod_status + until: _pod_status.stdout in ['green', 'yellow'] + retries: 60 + delay: 5 + changed_when: false diff --git a/roles/openshift_logging_elasticsearch/templates/es.j2 b/roles/openshift_logging_elasticsearch/templates/es.j2 index bf04094a3..4b189f255 100644 --- a/roles/openshift_logging_elasticsearch/templates/es.j2 +++ b/roles/openshift_logging_elasticsearch/templates/es.j2 @@ -17,6 +17,7 @@ spec: logging-infra: "{{logging_component}}" strategy: type: Recreate + triggers: [] template: metadata: name: "{{deploy_name}}" @@ -49,7 +50,7 @@ spec: - -provider=openshift - -client-id={{openshift_logging_elasticsearch_prometheus_sa}} - -client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token - - -cookie-secret={{ 16 | oo_random_word | b64encode }} + - -cookie-secret={{ 16 | lib_utils_oo_random_word | b64encode }} - -upstream=https://localhost:9200 - '-openshift-sar={"namespace": "{{ openshift_logging_elasticsearch_namespace}}", "verb": "view", "resource": "prometheus", "group": "metrics.openshift.io"}' - '-openshift-delegate-urls={"/": {"resource": "prometheus", "verb": "view", "group": "metrics.openshift.io", "namespace": "{{ openshift_logging_elasticsearch_namespace}}"}}' diff --git a/roles/openshift_logging_elasticsearch/vars/main.yml b/roles/openshift_logging_elasticsearch/vars/main.yml index 09e2ee4d0..122231031 100644 --- a/roles/openshift_logging_elasticsearch/vars/main.yml +++ b/roles/openshift_logging_elasticsearch/vars/main.yml @@ -1,9 +1,12 @@ --- -__latest_es_version: "3_6" -__allowed_es_versions: ["3_5", "3_6", "3_7"] +__latest_es_version: "3_9" +__allowed_es_versions: ["3_5", "3_6", "3_7", "3_8", "3_9"] __allowed_es_types: ["data-master", "data-client", "master", "client"] __es_log_appenders: ['file', 'console'] __kibana_index_modes: ["unique", "shared_ops"] +__es_version: "2.4.4" + +__es_local_curl: "curl -s --cacert /etc/elasticsearch/secret/admin-ca --cert /etc/elasticsearch/secret/admin-cert --key /etc/elasticsearch/secret/admin-key" # TODO: integrate these openshift_master_config_dir: "{{ openshift.common.config_base }}/master" @@ -12,3 +15,4 @@ es_min_masters_default: "{{ (openshift_logging_elasticsearch_replica_count | int es_min_masters: "{{ (openshift_logging_elasticsearch_replica_count == 1) | ternary(1, es_min_masters_default) }}" es_recover_after_nodes: "{{ openshift_logging_elasticsearch_replica_count | int }}" es_recover_expected_nodes: "{{ openshift_logging_elasticsearch_replica_count | int }}" +full_restart_cluster: False |