diff options
Diffstat (limited to 'roles/openshift_logging_elasticsearch/tasks')
5 files changed, 366 insertions, 181 deletions
diff --git a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml index c53a06019..a7cc8f0ec 100644 --- a/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/determine_version.yaml @@ -10,8 +10,14 @@ # should we just assume that we will have the correct major version? - set_fact: es_version="{{ openshift_logging_elasticsearch_image_version | regex_replace('^v?(?P<major>\d)\.(?P<minor>\d).*$', '3_\\g<minor>') }}" - when: openshift_logging_elasticsearch_image_version != 'latest' + when: + - openshift_logging_elasticsearch_image_version != 'latest' + - not openshift_logging_es5_techpreview | default(false) | bool - fail: msg: Invalid version specified for Elasticsearch - when: es_version not in __allowed_es_versions + when: + - es_version not in __allowed_es_versions + - not openshift_logging_es5_techpreview | default(false) | bool + +- include_tasks: get_es_version.yml diff --git a/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml new file mode 100644 index 000000000..7c4ecaa5b --- /dev/null +++ b/roles/openshift_logging_elasticsearch/tasks/get_es_version.yml @@ -0,0 +1,42 @@ +--- +- command: > + {{ openshift_client_binary }} get pod -l component=es,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name} + register: _cluster_pods + +- name: "Getting ES version for logging-es cluster" + command: > + {{ openshift_client_binary }} exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/' + register: _curl_output + when: _cluster_pods.stdout_lines | count > 0 + +- command: > + {{ openshift_client_binary }} get pod -l component=es-ops,provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name} + register: _ops_cluster_pods + +- name: "Getting ES version for logging-es-ops cluster" + command: > + {{ openshift_client_binary }} exec {{ _ops_cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XGET 'https://localhost:9200/' + register: _ops_curl_output + when: _ops_cluster_pods.stdout_lines | count > 0 + +- set_fact: + _es_output: "{{ _curl_output.stdout | from_json }}" + when: _curl_output.stdout is defined + +- set_fact: + _es_ops_output: "{{ _ops_curl_output.stdout | from_json }}" + when: _ops_curl_output.stdout is defined + +- set_fact: + _es_installed_version: "{{ _es_output.version.number }}" + when: + - _es_output is defined + - _es_output.version is defined + - _es_output.version.number is defined + +- set_fact: + _es_ops_installed_version: "{{ _es_ops_output.version.number }}" + when: + - _es_ops_output is defined + - _es_ops_output.version is defined + - _es_ops_output.version.number is defined diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml index 7790dc435..64e5a3a1f 100644 --- a/roles/openshift_logging_elasticsearch/tasks/main.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml @@ -1,4 +1,11 @@ --- +- name: Ensure that ElasticSearch has nodes to run on + fail: + msg: |- + No schedulable nodes found matching node selector for Elasticsearch - '{{ openshift_logging_es_nodeselector }}' + when: + - openshift_schedulable_node_labels | lib_utils_oo_has_no_matching_selector(openshift_logging_es_nodeselector) + - name: Validate Elasticsearch cluster size fail: msg="The openshift_logging_es_cluster_size may only be scaled down manually. Please see official documentation on how to do this." when: openshift_logging_facts.elasticsearch.deploymentconfigs | length > openshift_logging_es_cluster_size|int @@ -15,11 +22,11 @@ elasticsearch_name: "{{ 'logging-elasticsearch' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '')) }}" es_component: "{{ 'es' ~ ( (openshift_logging_elasticsearch_ops_deployment | default(false) | bool) | ternary('-ops', '') ) }}" -- name: Set default image variables based on deployment_type +- name: Set default image variables based on openshift_deployment_type include_vars: "{{ var_file_name }}" with_first_found: - - "{{ openshift_deployment_type | default(deployment_type) }}.yml" - - "default_images.yml" + - "{{ openshift_deployment_type }}.yml" + - "default_images.yml" loop_control: loop_var: var_file_name @@ -32,6 +39,18 @@ - include_tasks: determine_version.yaml +- set_fact: + full_restart_cluster: True + when: + - _es_installed_version is defined + - _es_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int + +- set_fact: + full_restart_cluster: True + when: + - _es_ops_installed_version is defined + - _es_ops_installed_version.split('.')[0] | int < __es_version.split('.')[0] | int + # allow passing in a tempdir - name: Create temp directory for doing work in command: mktemp -d /tmp/openshift-logging-ansible-XXXXXX @@ -52,7 +71,6 @@ # we want to make sure we have all the necessary components here # service account - - name: Create ES service account oc_serviceaccount: state: present @@ -67,22 +85,17 @@ name: "aggregated-logging-elasticsearch" namespace: "{{ openshift_logging_elasticsearch_namespace }}" when: - - openshift_logging_image_pull_secret == '' + - openshift_logging_image_pull_secret == '' # rolebinding reader -- copy: - src: rolebinding-reader.yml - dest: "{{ tempdir }}/rolebinding-reader.yml" - - name: Create rolebinding-reader role - oc_obj: + oc_clusterrole: state: present - name: "rolebinding-reader" - kind: clusterrole - namespace: "{{ openshift_logging_elasticsearch_namespace }}" - files: - - "{{ tempdir }}/rolebinding-reader.yml" - delete_after: true + name: rolebinding-reader + rules: + - apiGroups: [""] + resources: ["clusterrolebindings"] + verbs: ["get"] # SA roles - name: Set rolebinding-reader permissions for ES @@ -102,7 +115,7 @@ # logging-metrics-reader role - template: - src: logging-metrics-role.j2 + src: "{{ __base_file_dir }}/logging-metrics-role.j2" dest: "{{mktemp.stdout}}/templates/logging-metrics-role.yml" vars: namespace: "{{ openshift_logging_elasticsearch_namespace }}" @@ -122,21 +135,31 @@ - fail: msg: "There was an error creating the logging-metrics-role and binding: {{prometheus_out}}" when: - - "prometheus_out.stderr | length > 0" - - "'already exists' not in prometheus_out.stderr" + - "prometheus_out.stderr | length > 0" + - "'already exists' not in prometheus_out.stderr" + +- set_fact: + _logging_metrics_proxy_passwd: "{{ 16 | lib_utils_oo_random_word | b64encode }}" + +- template: + src: passwd.j2 + dest: "{{mktemp.stdout}}/passwd.yml" + vars: + logging_user_name: "{{ openshift_logging_elasticsearch_prometheus_sa }}" + logging_user_passwd: "{{ _logging_metrics_proxy_passwd }}" # View role and binding - name: Generate logging-elasticsearch-view-role template: - src: rolebinding.j2 + src: "{{ __base_file_dir }}/rolebinding.j2" dest: "{{mktemp.stdout}}/logging-elasticsearch-view-role.yaml" vars: obj_name: logging-elasticsearch-view-role roleRef: name: view subjects: - - kind: ServiceAccount - name: aggregated-logging-elasticsearch + - kind: ServiceAccount + name: aggregated-logging-elasticsearch changed_when: no - name: Set logging-elasticsearch-view-role role @@ -146,79 +169,108 @@ kind: rolebinding namespace: "{{ openshift_logging_elasticsearch_namespace }}" files: - - "{{ tempdir }}/logging-elasticsearch-view-role.yaml" + - "{{ tempdir }}/logging-elasticsearch-view-role.yaml" delete_after: true # configmap - assert: that: - - openshift_logging_elasticsearch_kibana_index_mode in __kibana_index_modes + - openshift_logging_elasticsearch_kibana_index_mode in __kibana_index_modes msg: "The openshift_logging_elasticsearch_kibana_index_mode '{{ openshift_logging_elasticsearch_kibana_index_mode }}' only supports one of: {{ __kibana_index_modes | join(', ') }}" - assert: that: - - "{{ openshift_logging_es_log_appenders | length > 0 }}" + - "{{ openshift_logging_es_log_appenders | length > 0 }}" msg: "The openshift_logging_es_log_appenders '{{ openshift_logging_es_log_appenders }}' has an unrecognized option and only supports the following as a list: {{ __es_log_appenders | join(', ') }}" - template: - src: elasticsearch-logging.yml.j2 - dest: "{{ tempdir }}/elasticsearch-logging.yml" - vars: - root_logger: "{{openshift_logging_es_log_appenders | join(', ')}}" - when: es_logging_contents is undefined - changed_when: no - -- set_fact: - __es_num_of_shards: "{{ _es_configmap | default({}) | walk('index.number_of_shards', '1') }}" - __es_num_of_replicas: "{{ _es_configmap | default({}) | walk('index.number_of_replicas', '0') }}" - -- template: - src: elasticsearch.yml.j2 + src: "{{ __base_file_dir }}/elasticsearch.yml.j2" dest: "{{ tempdir }}/elasticsearch.yml" vars: allow_cluster_reader: "{{ openshift_logging_elasticsearch_ops_allow_cluster_reader | lower | default('false') }}" - es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(None) or __es_num_of_shards }}" - es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas | default(None) or __es_num_of_replicas }}" + es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}" + es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}" es_kibana_index_mode: "{{ openshift_logging_elasticsearch_kibana_index_mode | default('unique') }}" - - when: es_config_contents is undefined - changed_when: no - -- copy: - content: "{{ es_logging_contents }}" - dest: "{{ tempdir }}/elasticsearch-logging.yml" - when: es_logging_contents is defined changed_when: no -- copy: - content: "{{ es_config_contents }}" - dest: "{{ tempdir }}/elasticsearch.yml" - when: es_config_contents is defined - changed_when: no - -- name: Set ES configmap - oc_configmap: - state: present - name: "{{ elasticsearch_name }}" - namespace: "{{ openshift_logging_elasticsearch_namespace }}" - from_file: - elasticsearch.yml: "{{ tempdir }}/elasticsearch.yml" - logging.yml: "{{ tempdir }}/elasticsearch-logging.yml" - register: es_config_creation - notify: "restart elasticsearch" +# create diff between current configmap files and our current files +- when: not openshift_logging_es5_techpreview + block: + - template: + src: "{{ __base_file_dir }}/elasticsearch-logging.yml.j2" + dest: "{{ tempdir }}/elasticsearch-logging.yml" + vars: + root_logger: "{{openshift_logging_es_log_appenders | join(', ')}}" + changed_when: no + + - include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-elasticsearch" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "elasticsearch.yml" + new_file: "{{ tempdir }}/elasticsearch.yml" + protected_lines: ["number_of_shards", "number_of_replicas"] + - current_file: "logging.yml" + new_file: "{{ tempdir }}/elasticsearch-logging.yml" + + - name: Set ES configmap + oc_configmap: + state: present + name: "{{ elasticsearch_name }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + from_file: + elasticsearch.yml: "{{ tempdir }}/elasticsearch.yml" + logging.yml: "{{ tempdir }}/elasticsearch-logging.yml" + register: es_config_creation + notify: "restart elasticsearch" + +- when: openshift_logging_es5_techpreview | bool + block: + - template: + src: "{{ __base_file_dir }}/log4j2.properties.j2" + dest: "{{ tempdir }}/log4j2.properties" + vars: + root_logger: "{{ openshift_logging_es_log_appenders | list }}" + changed_when: no + + - include_role: + name: openshift_logging + tasks_from: patch_configmap_files.yaml + vars: + configmap_name: "logging-elasticsearch" + configmap_namespace: "logging" + configmap_file_names: + - current_file: "elasticsearch.yml" + new_file: "{{ tempdir }}/elasticsearch.yml" + - current_file: "log4j2.properties" + new_file: "{{ tempdir }}/log4j2.properties" + + - name: Set ES configmap + oc_configmap: + state: present + name: "{{ elasticsearch_name }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + from_file: + elasticsearch.yml: "{{ tempdir }}/elasticsearch.yml" + log4j2.properties: "{{ tempdir }}/log4j2.properties" + register: es_config_creation + notify: "restart elasticsearch" - when: es_config_creation.changed | bool block: - - set_fact: - _restart_logging_components: "{{ _restart_logging_components | default([]) + [es_component] | unique }}" + - set_fact: + _restart_logging_components: "{{ _restart_logging_components | default([]) + [es_component] | unique }}" - - shell: > - oc get dc -l component="{{ es_component }}" -n "{{ openshift_logging_elasticsearch_namespace }}" -o name | cut -d'/' -f2 - register: _es_dcs + - shell: > + {{ openshift_client_binary }} get dc -l component="{{ es_component }}" -n "{{ openshift_logging_elasticsearch_namespace }}" -o name | cut -d'/' -f2 + register: _es_dcs - - set_fact: - _restart_logging_nodes: "{{ _restart_logging_nodes | default([]) + [_es_dcs.stdout] | unique }}" - when: _es_dcs.stdout != "" + - set_fact: + _restart_logging_nodes: "{{ _restart_logging_nodes | default([]) + [_es_dcs.stdout] | unique }}" + when: _es_dcs.stdout != "" # secret - name: Set ES secret @@ -227,22 +279,24 @@ name: "logging-elasticsearch" namespace: "{{ openshift_logging_elasticsearch_namespace }}" files: - - name: key - path: "{{ generated_certs_dir }}/logging-es.jks" - - name: truststore - path: "{{ generated_certs_dir }}/truststore.jks" - - name: searchguard.key - path: "{{ generated_certs_dir }}/elasticsearch.jks" - - name: searchguard.truststore - path: "{{ generated_certs_dir }}/truststore.jks" - - name: admin-key - path: "{{ generated_certs_dir }}/system.admin.key" - - name: admin-cert - path: "{{ generated_certs_dir }}/system.admin.crt" - - name: admin-ca - path: "{{ generated_certs_dir }}/ca.crt" - - name: admin.jks - path: "{{ generated_certs_dir }}/system.admin.jks" + - name: key + path: "{{ generated_certs_dir }}/logging-es.jks" + - name: truststore + path: "{{ generated_certs_dir }}/truststore.jks" + - name: searchguard.key + path: "{{ generated_certs_dir }}/elasticsearch.jks" + - name: searchguard.truststore + path: "{{ generated_certs_dir }}/truststore.jks" + - name: admin-key + path: "{{ generated_certs_dir }}/system.admin.key" + - name: admin-cert + path: "{{ generated_certs_dir }}/system.admin.crt" + - name: admin-ca + path: "{{ generated_certs_dir }}/ca.crt" + - name: admin.jks + path: "{{ generated_certs_dir }}/system.admin.jks" + - name: passwd.yml + path: "{{mktemp.stdout}}/passwd.yml" # services - name: Set logging-{{ es_component }}-cluster service @@ -256,7 +310,7 @@ labels: logging-infra: 'support' ports: - - port: 9300 + - port: 9300 - name: Set logging-{{ es_component }} service oc_service: @@ -269,8 +323,8 @@ labels: logging-infra: 'support' ports: - - port: 9200 - targetPort: "restapi" + - port: 9200 + targetPort: "restapi" - name: Set logging-{{ es_component}}-prometheus service oc_service: @@ -280,9 +334,9 @@ labels: logging-infra: 'support' ports: - - name: proxy - port: 443 - targetPort: 4443 + - name: proxy + port: 443 + targetPort: 4443 selector: component: "{{ es_component }}" provider: openshift @@ -310,46 +364,46 @@ # so we check for the presence of 'stderr' to determine if the obj exists or not # the RC for existing and not existing is both 0 - when: - - logging_elasticsearch_pvc.results.stderr is defined - - openshift_logging_elasticsearch_storage_type == "pvc" + - logging_elasticsearch_pvc.results.stderr is defined + - openshift_logging_elasticsearch_storage_type == "pvc" block: - # storageclasses are used by default but if static then disable - # storageclasses with the storageClassName set to "" in pvc.j2 - - name: Creating ES storage template - static - template: - src: pvc.j2 - dest: "{{ tempdir }}/templates/logging-es-pvc.yml" - vars: - obj_name: "{{ openshift_logging_elasticsearch_pvc_name }}" - size: "{{ (openshift_logging_elasticsearch_pvc_size | trim | length == 0) | ternary('10Gi', openshift_logging_elasticsearch_pvc_size) }}" - access_modes: "{{ openshift_logging_elasticsearch_pvc_access_modes | list }}" - pv_selector: "{{ openshift_logging_elasticsearch_pvc_pv_selector }}" - storage_class_name: "{{ openshift_logging_elasticsearch_pvc_storage_class_name | default('', true) }}" - when: - - not openshift_logging_elasticsearch_pvc_dynamic | bool - - # Storageclasses are used by default if configured - - name: Creating ES storage template - dynamic - template: - src: pvc.j2 - dest: "{{ tempdir }}/templates/logging-es-pvc.yml" - vars: - obj_name: "{{ openshift_logging_elasticsearch_pvc_name }}" - size: "{{ (openshift_logging_elasticsearch_pvc_size | trim | length == 0) | ternary('10Gi', openshift_logging_elasticsearch_pvc_size) }}" - access_modes: "{{ openshift_logging_elasticsearch_pvc_access_modes | list }}" - pv_selector: "{{ openshift_logging_elasticsearch_pvc_pv_selector }}" - when: - - openshift_logging_elasticsearch_pvc_dynamic | bool - - - name: Set ES storage - oc_obj: - state: present - kind: pvc - name: "{{ openshift_logging_elasticsearch_pvc_name }}" - namespace: "{{ openshift_logging_elasticsearch_namespace }}" - files: - - "{{ tempdir }}/templates/logging-es-pvc.yml" - delete_after: true + # storageclasses are used by default but if static then disable + # storageclasses with the storageClassName set to "" in pvc.j2 + - name: Creating ES storage template - static + template: + src: "{{ __base_file_dir }}/pvc.j2" + dest: "{{ tempdir }}/templates/logging-es-pvc.yml" + vars: + obj_name: "{{ openshift_logging_elasticsearch_pvc_name }}" + size: "{{ (openshift_logging_elasticsearch_pvc_size | trim | length == 0) | ternary('10Gi', openshift_logging_elasticsearch_pvc_size) }}" + access_modes: "{{ openshift_logging_elasticsearch_pvc_access_modes | list }}" + pv_selector: "{{ openshift_logging_elasticsearch_pvc_pv_selector }}" + storage_class_name: "{{ openshift_logging_elasticsearch_pvc_storage_class_name | default('', true) }}" + when: + - not openshift_logging_elasticsearch_pvc_dynamic | bool + + # Storageclasses are used by default if configured + - name: Creating ES storage template - dynamic + template: + src: "{{ __base_file_dir }}/pvc.j2" + dest: "{{ tempdir }}/templates/logging-es-pvc.yml" + vars: + obj_name: "{{ openshift_logging_elasticsearch_pvc_name }}" + size: "{{ (openshift_logging_elasticsearch_pvc_size | trim | length == 0) | ternary('10Gi', openshift_logging_elasticsearch_pvc_size) }}" + access_modes: "{{ openshift_logging_elasticsearch_pvc_access_modes | list }}" + pv_selector: "{{ openshift_logging_elasticsearch_pvc_pv_selector }}" + when: + - openshift_logging_elasticsearch_pvc_dynamic | bool + + - name: Set ES storage + oc_obj: + state: present + kind: pvc + name: "{{ openshift_logging_elasticsearch_pvc_name }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + files: + - "{{ tempdir }}/templates/logging-es-pvc.yml" + delete_after: true - set_fact: es_deploy_name: "logging-{{ es_component }}-{{ openshift_logging_elasticsearch_deployment_type }}-{{ 8 | lib_utils_oo_random_word('abcdefghijklmnopqrstuvwxyz0123456789') }}" @@ -362,7 +416,7 @@ # DC - name: Set ES dc templates template: - src: es.j2 + src: "{{ __base_file_dir }}/es.j2" dest: "{{ tempdir }}/templates/logging-es-dc.yml" vars: es_cluster_name: "{{ es_component }}" @@ -379,6 +433,9 @@ es_container_security_context: "{{ _es_containers.elasticsearch.securityContext if _es_containers is defined and 'elasticsearch' in _es_containers and 'securityContext' in _es_containers.elasticsearch else None }}" deploy_type: "{{ openshift_logging_elasticsearch_deployment_type }}" es_replicas: 1 + basic_auth_passwd: "{{ _logging_metrics_proxy_passwd | b64decode }}" + es_number_of_shards: "{{ openshift_logging_es_number_of_shards | default(1) }}" + es_number_of_replicas: "{{ openshift_logging_es_number_of_replicas| default(0) }}" - name: Set ES dc oc_obj: @@ -387,7 +444,7 @@ namespace: "{{ openshift_logging_elasticsearch_namespace }}" kind: dc files: - - "{{ tempdir }}/templates/logging-es-dc.yml" + - "{{ tempdir }}/templates/logging-es-dc.yml" delete_after: true register: es_dc_creation notify: "restart elasticsearch" @@ -402,42 +459,42 @@ src: "{{ generated_certs_dir }}/{{ item.file }}" register: key_pairs with_items: - - { name: "ca_file", file: "ca.crt" } - - { name: "es_key", file: "system.logging.es.key" } - - { name: "es_cert", file: "system.logging.es.crt" } + - { name: "ca_file", file: "ca.crt" } + - { name: "es_key", file: "system.logging.es.key" } + - { name: "es_cert", file: "system.logging.es.crt" } when: openshift_logging_es_allow_external | bool - set_fact: es_key: "{{ lookup('file', openshift_logging_es_key) | b64encode }}" when: - - openshift_logging_es_key | trim | length > 0 - - openshift_logging_es_allow_external | bool + - openshift_logging_es_key | trim | length > 0 + - openshift_logging_es_allow_external | bool changed_when: false - set_fact: es_cert: "{{ lookup('file', openshift_logging_es_cert) | b64encode }}" when: - - openshift_logging_es_cert | trim | length > 0 - - openshift_logging_es_allow_external | bool + - openshift_logging_es_cert | trim | length > 0 + - openshift_logging_es_allow_external | bool changed_when: false - set_fact: es_ca: "{{ lookup('file', openshift_logging_es_ca_ext) | b64encode }}" when: - - openshift_logging_es_ca_ext | trim | length > 0 - - openshift_logging_es_allow_external | bool + - openshift_logging_es_ca_ext | trim | length > 0 + - openshift_logging_es_allow_external | bool changed_when: false - set_fact: es_ca: "{{ key_pairs | entry_from_named_pair('ca_file') }}" when: - - es_ca is not defined - - openshift_logging_es_allow_external | bool + - es_ca is not defined + - openshift_logging_es_allow_external | bool changed_when: false - name: Generating Elasticsearch {{ es_component }} route template template: - src: route_reencrypt.j2 + src: "{{ __base_file_dir }}/route_reencrypt.j2" dest: "{{mktemp.stdout}}/templates/logging-{{ es_component }}-route.yaml" vars: obj_name: "logging-{{ es_component }}" @@ -463,7 +520,7 @@ namespace: "{{ openshift_logging_elasticsearch_namespace }}" kind: route files: - - "{{ tempdir }}/templates/logging-{{ es_component }}-route.yaml" + - "{{ tempdir }}/templates/logging-{{ es_component }}-route.yaml" when: openshift_logging_es_allow_external | bool ## Placeholder for migration when necessary ## diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml index 4a32453e3..14f2313e1 100644 --- a/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml +++ b/roles/openshift_logging_elasticsearch/tasks/restart_cluster.yml @@ -1,35 +1,113 @@ --- ## get all pods for the cluster - command: > - oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + {{ openshift_client_binary }} get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name} register: _cluster_pods -- name: "Disable shard balancing for logging-{{ _cluster_component }} cluster" - command: > - oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "none" } }' - register: _disable_output - changed_when: "'\"acknowledged\":true' in _disable_output.stdout" +### Check for cluster state before making changes -- if its red then we don't want to continue +- name: "Checking current health for {{ _es_node }} cluster" + shell: > + {{ openshift_client_binary }} exec "{{ _cluster_pods.stdout.split(' ')[0] }}" -c elasticsearch -n "{{ openshift_logging_elasticsearch_namespace }}" -- es_cluster_health + register: _pod_status when: _cluster_pods.stdout_lines | count > 0 -- command: > - oc get dc -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} - register: _cluster_dcs - -## restart the node if it's dc is in the list of nodes to restart? -- name: "Restart ES node {{ _es_node }}" - include_tasks: restart_es_node.yml - with_items: "{{ _restart_logging_nodes }}" - loop_control: - loop_var: _es_node - when: _es_node in _cluster_dcs.stdout - -## we may need a new first pod to run against -- fetch them all again -- command: > - oc get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} - register: _cluster_pods +- when: + - _pod_status.stdout is defined + - (_pod_status.stdout | from_json)['status'] in ['red'] + block: + - name: Set Logging message to manually restart + run_once: true + set_stats: + data: + installer_phase_logging: + message: "Cluster logging-{{ _cluster_component }} was in a red state and will not be automatically restarted. Please see documentation regarding doing a {{ 'full' if full_restart_cluster | bool else 'rolling'}} cluster restart." + + - debug: msg="Cluster logging-{{ _cluster_component }} was in a red state and will not be automatically restarted. Please see documentation regarding doing a {{ 'full' if full_restart_cluster | bool else 'rolling'}} cluster restart." + +- when: _pod_status.stdout is undefined or (_pod_status.stdout | from_json)['status'] in ['green', 'yellow'] + block: + # Disable external communication for {{ _cluster_component }} + - name: Disable external communication for logging-{{ _cluster_component }} + oc_service: + state: present + name: "logging-{{ _cluster_component }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + selector: + component: "{{ _cluster_component }}" + provider: openshift + connection: blocked + labels: + logging-infra: 'support' + ports: + - port: 9200 + targetPort: "restapi" + when: + - full_restart_cluster | bool + + - name: "Disable shard balancing for logging-{{ _cluster_component }} cluster" + command: > + {{ openshift_client_binary }} exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "none" } }' + register: _disable_output + changed_when: "'\"acknowledged\":true' in _disable_output.stdout" + when: _cluster_pods.stdout_lines | count > 0 + + # Flush ES + - name: "Flushing for logging-{{ _cluster_component }} cluster" + command: > + {{ openshift_client_binary }} exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_flush/synced' + register: _flush_output + changed_when: "'\"acknowledged\":true' in _flush_output.stdout" + when: + - _cluster_pods.stdout_lines | count > 0 + - full_restart_cluster | bool + + - command: > + {{ openshift_client_binary }} get dc -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + register: _cluster_dcs + + ## restart all dcs for full restart + - name: "Restart ES node {{ _es_node }}" + include_tasks: restart_es_node.yml + with_items: "{{ _cluster_dcs }}" + loop_control: + loop_var: _es_node + when: + - full_restart_cluster | bool + + ## restart the node if it's dc is in the list of nodes to restart? + - name: "Restart ES node {{ _es_node }}" + include_tasks: restart_es_node.yml + with_items: "{{ _restart_logging_nodes }}" + loop_control: + loop_var: _es_node + when: + - not full_restart_cluster | bool + - _es_node in _cluster_dcs.stdout + + ## we may need a new first pod to run against -- fetch them all again + - command: > + {{ openshift_client_binary }} get pod -l component={{ _cluster_component }},provider=openshift -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[?(@.status.phase==\"Running\")].metadata.name} + register: _cluster_pods + + - name: "Enable shard balancing for logging-{{ _cluster_component }} cluster" + command: > + {{ openshift_client_binary }} exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "all" } }' + register: _enable_output + changed_when: "'\"acknowledged\":true' in _enable_output.stdout" -- name: "Enable shard balancing for logging-{{ _cluster_component }} cluster" - command: > - oc exec {{ _cluster_pods.stdout.split(' ')[0] }} -c elasticsearch -n {{ openshift_logging_elasticsearch_namespace }} -- {{ __es_local_curl }} -XPUT 'https://localhost:9200/_cluster/settings' -d '{ "transient": { "cluster.routing.allocation.enable" : "all" } }' - register: _enable_output - changed_when: "'\"acknowledged\":true' in _enable_output.stdout" + # Reenable external communication for {{ _cluster_component }} + - name: Reenable external communication for logging-{{ _cluster_component }} + oc_service: + state: present + name: "logging-{{ _cluster_component }}" + namespace: "{{ openshift_logging_elasticsearch_namespace }}" + selector: + component: "{{ _cluster_component }}" + provider: openshift + labels: + logging-infra: 'support' + ports: + - port: 9200 + targetPort: "restapi" + when: + - full_restart_cluster | bool diff --git a/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml index b07b232ce..a1e172168 100644 --- a/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml +++ b/roles/openshift_logging_elasticsearch/tasks/restart_es_node.yml @@ -1,7 +1,7 @@ --- - name: "Rolling out new pod(s) for {{ _es_node }}" command: > - oc rollout latest {{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} + {{ openshift_client_binary }} rollout latest {{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} - name: "Waiting for {{ _es_node }} to finish scaling up" oc_obj: @@ -14,22 +14,24 @@ - _dc_output.results.results[0].status is defined - _dc_output.results.results[0].status.readyReplicas is defined - _dc_output.results.results[0].status.readyReplicas > 0 + - _dc_output.results.results[0].status.updatedReplicas is defined + - _dc_output.results.results[0].status.updatedReplicas > 0 retries: 60 delay: 30 - name: Gettings name(s) of replica pod(s) command: > - oc get pods -l deploymentconfig={{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} + {{ openshift_client_binary }} get pods -l deploymentconfig={{ _es_node }} -n {{ openshift_logging_elasticsearch_namespace }} -o jsonpath={.items[*].metadata.name} register: _pods - name: "Waiting for ES to be ready for {{ _es_node }}" shell: > - oc exec "{{ _pod }}" -c elasticsearch -n "{{ openshift_logging_elasticsearch_namespace }}" -- {{ __es_local_curl }} https://localhost:9200/_cat/health | cut -d' ' -f4 + {{ openshift_client_binary }} exec "{{ _pod }}" -c elasticsearch -n "{{ openshift_logging_elasticsearch_namespace }}" -- es_cluster_health with_items: "{{ _pods.stdout.split(' ') }}" loop_control: loop_var: _pod register: _pod_status - until: _pod_status.stdout in ['green', 'yellow'] + until: (_pod_status.stdout | from_json)['status'] in ['green', 'yellow'] retries: 60 delay: 5 changed_when: false |