From e1df3b6ba7b78255c8022468e9ad1a32255e5e59 Mon Sep 17 00:00:00 2001 From: Michael Gugino Date: Fri, 8 Dec 2017 13:38:03 -0500 Subject: Refactor upgrade codepaths step 1 This commit refactors some upgrade code paths. Touched areas are: 1) Reduces usage of 'oo_all_hosts' in various places, especially when running upgrade_control_plane. 2) Reuses common code across the various upgrade* playbooks. 3) Moves docker upgrade checks into container_runtime_role. 4) Combines smaller playbooks and plays to reduce file sprawl. --- .../openshift-cluster/upgrades/pre/config.yml | 77 ++++++++++++++++++ .../openshift-cluster/upgrades/pre/gate_checks.yml | 6 -- .../pre/tasks/verify_docker_upgrade_targets.yml | 22 ----- .../upgrades/pre/verify_cluster.yml | 93 ++++++++++++++++++++++ .../upgrades/pre/verify_control_plane_running.yml | 37 --------- .../upgrades/pre/verify_etcd3_backend.yml | 22 ----- .../upgrades/pre/verify_health_checks.yml | 16 ---- .../upgrades/pre/verify_inventory_vars.yml | 37 --------- 8 files changed, 170 insertions(+), 140 deletions(-) create mode 100644 playbooks/common/openshift-cluster/upgrades/pre/config.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/tasks/verify_docker_upgrade_targets.yml create mode 100644 playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/verify_etcd3_backend.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/verify_health_checks.yml delete mode 100644 playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml (limited to 'playbooks/common/openshift-cluster/upgrades/pre') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/config.yml b/playbooks/common/openshift-cluster/upgrades/pre/config.yml new file mode 100644 index 000000000..d5b82d9a0 --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/config.yml @@ -0,0 +1,77 @@ +--- + +# Pre-upgrade +- import_playbook: ../initialize_nodes_to_upgrade.yml + +- import_playbook: verify_cluster.yml + +- name: Update repos on upgrade hosts + hosts: "{{ l_upgrade_repo_hosts }}" + roles: + - openshift_repos + +- name: Set openshift_no_proxy_internal_hostnames + hosts: "{{ l_upgrade_no_proxy_hosts }}" + tasks: + - set_fact: + openshift_no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config'] + | union(groups['oo_masters_to_config']) + | union(groups['oo_etcd_to_config'] | default([]))) + | oo_collect('openshift.common.hostname') | default([]) | join (',') + }}" + when: + - openshift_http_proxy is defined or openshift_https_proxy is defined + - openshift_generate_no_proxy_hosts | default(True) | bool + +- name: OpenShift Health Checks + hosts: "{{ l_upgrade_health_check_hosts }}" + any_errors_fatal: true + roles: + - openshift_health_checker + vars: + - r_openshift_health_checker_playbook_context: upgrade + post_tasks: + - name: Run health checks (upgrade) + action: openshift_health_check + args: + checks: + - disk_availability + - memory_availability + - docker_image_availability + +- import_playbook: ../disable_excluders.yml + +- import_playbook: ../../../../init/version.yml + vars: + # Request specific openshift_release and let the openshift_version role handle converting this + # to a more specific version, respecting openshift_image_tag and openshift_pkg_version if + # defined, and overriding the normal behavior of protecting the installed version + openshift_release: "{{ openshift_upgrade_target }}" + openshift_protect_installed_version: False + +# If we're only upgrading nodes, we need to ensure masters are already upgraded +- name: Verify masters are already upgraded + hosts: oo_masters_to_config + tasks: + - fail: msg="Master running {{ openshift.common.version }} must be upgraded to {{ openshift_version }} before node upgrade can be run." + when: + - l_upgrade_nodes_only | default(False) | bool + - openshift.common.version != openshift_version + +# If we're only upgrading nodes, skip this. +- import_playbook: ../../../../openshift-master/private/validate_restart.yml + when: not (l_upgrade_nodes_only | default(False)) | bool + +- name: Verify upgrade targets + hosts: "{{ l_upgrade_verify_targets_hosts }}" + roles: + - role: openshift_facts + tasks: + - include_tasks: verify_upgrade_targets.yml + +- name: Verify docker upgrade targets + hosts: "{{ l_upgrade_docker_target_hosts }}" + tasks: + - include_role: + name: container_runtime + tasks_from: docker_upgrade_check.yml diff --git a/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml b/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml deleted file mode 100644 index 8ecae4539..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/gate_checks.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Flag pre-upgrade checks complete for hosts without errors - hosts: oo_masters_to_config:oo_nodes_to_upgrade:oo_etcd_to_config - tasks: - - set_fact: - pre_upgrade_complete: True diff --git a/playbooks/common/openshift-cluster/upgrades/pre/tasks/verify_docker_upgrade_targets.yml b/playbooks/common/openshift-cluster/upgrades/pre/tasks/verify_docker_upgrade_targets.yml deleted file mode 100644 index 18a08eb99..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/tasks/verify_docker_upgrade_targets.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Only check if docker upgrade is required if docker_upgrade is not -# already set to False. -- include_tasks: ../../docker/upgrade_check.yml - when: - - docker_upgrade is not defined or (docker_upgrade | bool) - - not (openshift.common.is_atomic | bool) - -# Additional checks for Atomic hosts: - -- name: Determine available Docker - shell: "rpm -q --queryformat '---\ncurr_version: %{VERSION}\navail_version: \n' docker" - register: g_atomic_docker_version_result - when: openshift.common.is_atomic | bool - -- set_fact: - l_docker_version: "{{ g_atomic_docker_version_result.stdout | from_yaml }}" - when: openshift.common.is_atomic | bool - -- fail: - msg: This playbook requires access to Docker 1.12 or later - when: openshift.common.is_atomic | bool and l_docker_version.avail_version | default(l_docker_version.curr_version, true) | version_compare('1.12','<') diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml new file mode 100644 index 000000000..2ab9f852c --- /dev/null +++ b/playbooks/common/openshift-cluster/upgrades/pre/verify_cluster.yml @@ -0,0 +1,93 @@ +--- +# Verify a few items before we proceed with upgrade process. + +- name: Verify upgrade can proceed on first master + hosts: oo_first_master + gather_facts: no + tasks: + - fail: + msg: > + This upgrade is only supported for origin and openshift-enterprise + deployment types + when: deployment_type not in ['origin','openshift-enterprise'] + + # Error out in situations where the user has older versions specified in their + # inventory in any of the openshift_release, openshift_image_tag, and + # openshift_pkg_version variables. These must be removed or updated to proceed + # with upgrade. + # TODO: Should we block if you're *over* the next major release version as well? + - fail: + msg: > + openshift_pkg_version is {{ openshift_pkg_version }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(openshift_upgrade_target ,'<') + + - fail: + msg: > + openshift_image_tag is {{ openshift_image_tag }} which is not a + valid version for a {{ openshift_upgrade_target }} upgrade + when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(openshift_upgrade_target ,'<') + + - set_fact: + openshift_release: "{{ openshift_release[1:] }}" + when: openshift_release is defined and openshift_release[0] == 'v' + + - fail: + msg: > + openshift_release is {{ openshift_release }} which is not a + valid release for a {{ openshift_upgrade_target }} upgrade + when: openshift_release is defined and not openshift_release | version_compare(openshift_upgrade_target ,'=') + +- name: Verify master processes + hosts: oo_masters_to_config + roles: + - lib_utils + - openshift_facts + tasks: + - name: Read master storage backend setting + yedit: + state: list + src: /etc/origin/master/master-config.yaml + key: kubernetesMasterConfig.apiServerArguments.storage-backend + register: _storage_backend + + - fail: + msg: "Storage backend in /etc/origin/master/master-config.yaml must be set to 'etcd3' before the upgrade can continue" + when: + # assuming the master-config.yml is properly configured, i.e. the value is a list + - _storage_backend.result | default([], true) | length == 0 or _storage_backend.result[0] != "etcd3" + + - debug: + msg: "Storage backend is set to etcd3" + + - openshift_facts: + role: master + local_facts: + ha: "{{ groups.oo_masters_to_config | length > 1 }}" + + - when: openshift.common.is_containerized | bool + block: + - set_fact: + master_services: + - "{{ openshift_service_type }}-master" + + # In case of the non-ha to ha upgrade. + - name: Check if the {{ openshift_service_type }}-master-api.service exists + command: > + systemctl list-units {{ openshift_service_type }}-master-api.service --no-legend + register: master_api_service_status + + - set_fact: + master_services: + - "{{ openshift_service_type }}-master-api" + - "{{ openshift_service_type }}-master-controllers" + when: + - master_api_service_status.stdout_lines | length > 0 + - (openshift_service_type + '-master-api.service') in master_api_service_status.stdout_lines[0] + + - name: Ensure Master is running + service: + name: "{{ item }}" + state: started + enabled: yes + with_items: "{{ master_services }}" diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml deleted file mode 100644 index bef95546d..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/verify_control_plane_running.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Verify master processes - hosts: oo_masters_to_config - roles: - - openshift_facts - tasks: - - openshift_facts: - role: master - local_facts: - ha: "{{ groups.oo_masters_to_config | length > 1 }}" - - - when: openshift.common.is_containerized | bool - block: - - set_fact: - master_services: - - "{{ openshift_service_type }}-master" - - # In case of the non-ha to ha upgrade. - - name: Check if the {{ openshift_service_type }}-master-api.service exists - command: > - systemctl list-units {{ openshift_service_type }}-master-api.service --no-legend - register: master_api_service_status - - - set_fact: - master_services: - - "{{ openshift_service_type }}-master-api" - - "{{ openshift_service_type }}-master-controllers" - when: - - master_api_service_status.stdout_lines | length > 0 - - (openshift_service_type + '-master-api.service') in master_api_service_status.stdout_lines[0] - - - name: Ensure Master is running - service: - name: "{{ item }}" - state: started - enabled: yes - with_items: "{{ master_services }}" diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_etcd3_backend.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_etcd3_backend.yml deleted file mode 100644 index f75ae3b15..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/verify_etcd3_backend.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Verify all masters has etcd3 storage backend set - hosts: oo_masters_to_config - gather_facts: no - roles: - - lib_utils - tasks: - - name: Read master storage backend setting - yedit: - state: list - src: /etc/origin/master/master-config.yaml - key: kubernetesMasterConfig.apiServerArguments.storage-backend - register: _storage_backend - - - fail: - msg: "Storage backend in /etc/origin/master/master-config.yaml must be set to 'etcd3' before the upgrade can continue" - when: - # assuming the master-config.yml is properly configured, i.e. the value is a list - - _storage_backend.result | default([], true) | length == 0 or _storage_backend.result[0] != "etcd3" - - - debug: - msg: "Storage backend is set to etcd3" diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_health_checks.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_health_checks.yml deleted file mode 100644 index 2a8de50a2..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/verify_health_checks.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: OpenShift Health Checks - hosts: oo_all_hosts - any_errors_fatal: true - roles: - - openshift_health_checker - vars: - - r_openshift_health_checker_playbook_context: upgrade - post_tasks: - - name: Run health checks (upgrade) - action: openshift_health_check - args: - checks: - - disk_availability - - memory_availability - - docker_image_availability diff --git a/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml b/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml deleted file mode 100644 index 3c0017891..000000000 --- a/playbooks/common/openshift-cluster/upgrades/pre/verify_inventory_vars.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Verify upgrade can proceed on first master - hosts: oo_first_master - gather_facts: no - tasks: - - fail: - msg: > - This upgrade is only supported for origin and openshift-enterprise - deployment types - when: deployment_type not in ['origin','openshift-enterprise'] - - # Error out in situations where the user has older versions specified in their - # inventory in any of the openshift_release, openshift_image_tag, and - # openshift_pkg_version variables. These must be removed or updated to proceed - # with upgrade. - # TODO: Should we block if you're *over* the next major release version as well? - - fail: - msg: > - openshift_pkg_version is {{ openshift_pkg_version }} which is not a - valid version for a {{ openshift_upgrade_target }} upgrade - when: openshift_pkg_version is defined and openshift_pkg_version.split('-',1).1 | version_compare(openshift_upgrade_target ,'<') - - - fail: - msg: > - openshift_image_tag is {{ openshift_image_tag }} which is not a - valid version for a {{ openshift_upgrade_target }} upgrade - when: openshift_image_tag is defined and openshift_image_tag.split('v',1).1 | version_compare(openshift_upgrade_target ,'<') - - - set_fact: - openshift_release: "{{ openshift_release[1:] }}" - when: openshift_release is defined and openshift_release[0] == 'v' - - - fail: - msg: > - openshift_release is {{ openshift_release }} which is not a - valid release for a {{ openshift_upgrade_target }} upgrade - when: openshift_release is defined and not openshift_release | version_compare(openshift_upgrade_target ,'=') -- cgit v1.2.1