From 112e1696cc7d2c8c5b11682b584b45a37e947099 Mon Sep 17 00:00:00 2001 From: Russell Teague Date: Tue, 5 Dec 2017 16:30:06 -0500 Subject: Playbook Consolidation - etcd Upgrade --- playbooks/openshift-etcd/private/ca.yml | 2 +- .../openshift-etcd/private/certificates-backup.yml | 6 +- .../openshift-etcd/private/embedded2external.yml | 22 ++++---- playbooks/openshift-etcd/private/migrate.yml | 14 ++--- playbooks/openshift-etcd/private/redeploy-ca.yml | 12 ++-- .../private/redeploy-certificates.yml | 4 +- playbooks/openshift-etcd/private/scaleup.yml | 4 +- .../openshift-etcd/private/server_certificates.yml | 2 +- .../openshift-etcd/private/upgrade_backup.yml | 28 ++++++++++ .../private/upgrade_image_members.yml | 17 ++++++ playbooks/openshift-etcd/private/upgrade_main.yml | 29 ++++++++++ .../openshift-etcd/private/upgrade_rpm_members.yml | 18 ++++++ playbooks/openshift-etcd/private/upgrade_step.yml | 64 ++++++++++++++++++++++ playbooks/openshift-etcd/redeploy-ca.yml | 4 +- playbooks/openshift-etcd/redeploy-certificates.yml | 8 +-- playbooks/openshift-etcd/upgrade.yml | 4 ++ 16 files changed, 199 insertions(+), 39 deletions(-) create mode 100644 playbooks/openshift-etcd/private/upgrade_backup.yml create mode 100644 playbooks/openshift-etcd/private/upgrade_image_members.yml create mode 100644 playbooks/openshift-etcd/private/upgrade_main.yml create mode 100644 playbooks/openshift-etcd/private/upgrade_rpm_members.yml create mode 100644 playbooks/openshift-etcd/private/upgrade_step.yml create mode 100644 playbooks/openshift-etcd/upgrade.yml (limited to 'playbooks/openshift-etcd') diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml index c9f186e72..f3bb3c2d1 100644 --- a/playbooks/openshift-etcd/private/ca.yml +++ b/playbooks/openshift-etcd/private/ca.yml @@ -7,7 +7,7 @@ tasks: - include_role: name: etcd - tasks_from: ca + tasks_from: ca.yml vars: etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" diff --git a/playbooks/openshift-etcd/private/certificates-backup.yml b/playbooks/openshift-etcd/private/certificates-backup.yml index d738c8207..ce21a1f96 100644 --- a/playbooks/openshift-etcd/private/certificates-backup.yml +++ b/playbooks/openshift-etcd/private/certificates-backup.yml @@ -5,10 +5,10 @@ tasks: - include_role: name: etcd - tasks_from: backup_generated_certificates + tasks_from: backup_generated_certificates.yml - include_role: name: etcd - tasks_from: remove_generated_certificates + tasks_from: remove_generated_certificates.yml - name: Backup deployed etcd certificates hosts: oo_etcd_to_config @@ -16,4 +16,4 @@ tasks: - include_role: name: etcd - tasks_from: backup_server_certificates + tasks_from: backup_server_certificates.yml diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml index 514319b88..c7a532622 100644 --- a/playbooks/openshift-etcd/private/embedded2external.yml +++ b/playbooks/openshift-etcd/private/embedded2external.yml @@ -20,7 +20,7 @@ - name: Check the master API is ready include_role: name: openshift_master - tasks_from: check_master_api_is_ready + tasks_from: check_master_api_is_ready.yml - set_fact: master_service: "{{ openshift.common.service_type + '-master' }}" embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" @@ -34,7 +34,7 @@ # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285 - include_role: name: etcd - tasks_from: backup + tasks_from: backup.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_embedded_etcd: "{{ true }}" @@ -42,7 +42,7 @@ - include_role: name: etcd - tasks_from: backup.archive + tasks_from: backup.archive.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_embedded_etcd: "{{ true }}" @@ -58,7 +58,7 @@ tasks: - include_role: name: etcd - tasks_from: backup_master_etcd_certificates + tasks_from: backup_master_etcd_certificates.yml - name: Redeploy master etcd certificates import_playbook: master_etcd_certificates.yml @@ -75,10 +75,10 @@ pre_tasks: - include_role: name: etcd - tasks_from: disable_etcd + tasks_from: disable_etcd.yml - include_role: name: etcd - tasks_from: clean_data + tasks_from: clean_data.yml # 6. copy the embedded etcd backup to the external host # TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory @@ -93,7 +93,7 @@ - include_role: name: etcd - tasks_from: backup.fetch + tasks_from: backup.fetch.yml vars: etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" r_etcd_common_backup_tag: pre-migrate @@ -103,7 +103,7 @@ - include_role: name: etcd - tasks_from: backup.copy + tasks_from: backup.copy.yml vars: etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" r_etcd_common_backup_tag: pre-migrate @@ -124,14 +124,14 @@ tasks: - include_role: name: etcd - tasks_from: backup.unarchive + tasks_from: backup.unarchive.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - include_role: name: etcd - tasks_from: backup.force_new_cluster + tasks_from: backup.force_new_cluster.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" @@ -145,7 +145,7 @@ tasks: - include_role: name: openshift_master - tasks_from: configure_external_etcd + tasks_from: configure_external_etcd.yml vars: etcd_peer_url_scheme: "https" etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}" diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml index 4269918c2..834bd242d 100644 --- a/playbooks/openshift-etcd/private/migrate.yml +++ b/playbooks/openshift-etcd/private/migrate.yml @@ -17,7 +17,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.pre_check + tasks_from: migrate.pre_check.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ ansible_default_ipv4.address }}" @@ -46,7 +46,7 @@ post_tasks: - include_role: name: etcd - tasks_from: backup + tasks_from: backup.yml vars: r_etcd_common_backup_tag: pre-migration r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" @@ -74,7 +74,7 @@ pre_tasks: - include_role: name: etcd - tasks_from: disable_etcd + tasks_from: disable_etcd.yml - name: Migrate data on first etcd hosts: oo_etcd_to_migrate[0] @@ -82,7 +82,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate + tasks_from: migrate.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ openshift.common.ip }}" @@ -95,7 +95,7 @@ tasks: - include_role: name: etcd - tasks_from: clean_data + tasks_from: clean_data.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ openshift.common.ip }}" @@ -132,7 +132,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.add_ttls + tasks_from: migrate.add_ttls.yml vars: etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}" etcd_url_scheme: "https" @@ -144,7 +144,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.configure_master + tasks_from: migrate.configure_master.yml when: etcd_migration_failed | length == 0 - debug: msg: "Skipping master re-configuration since migration failed." diff --git a/playbooks/openshift-etcd/private/redeploy-ca.yml b/playbooks/openshift-etcd/private/redeploy-ca.yml index cc5d57031..158bcb849 100644 --- a/playbooks/openshift-etcd/private/redeploy-ca.yml +++ b/playbooks/openshift-etcd/private/redeploy-ca.yml @@ -16,12 +16,12 @@ tasks: - include_role: name: etcd - tasks_from: backup_ca_certificates + tasks_from: backup_ca_certificates.yml - include_role: name: etcd - tasks_from: remove_ca_certificates + tasks_from: remove_ca_certificates.yml -- include: ca.yml +- import_playbook: ca.yml - name: Create temp directory for syncing certs hosts: localhost @@ -44,7 +44,7 @@ etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" -- include: restart.yml +- import_playbook: restart.yml # Do not restart etcd when etcd certificates were previously expired. when: ('expired' not in (hostvars | oo_select_keys(groups['etcd']) @@ -56,7 +56,7 @@ tasks: - include_role: name: etcd - tasks_from: retrieve_ca_certificates + tasks_from: retrieve_ca_certificates.yml vars: etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" @@ -82,7 +82,7 @@ state: absent changed_when: false -- include: ../../openshift-master/private/restart.yml +- import_playbook: ../../openshift-master/private/restart.yml # Do not restart masters when master or etcd certificates were previously expired. when: # masters diff --git a/playbooks/openshift-etcd/private/redeploy-certificates.yml b/playbooks/openshift-etcd/private/redeploy-certificates.yml index cc1e6adf5..1c8eb27ac 100644 --- a/playbooks/openshift-etcd/private/redeploy-certificates.yml +++ b/playbooks/openshift-etcd/private/redeploy-certificates.yml @@ -11,8 +11,8 @@ # certificates were previously expired. - role: openshift_certificate_expiry -- include: certificates-backup.yml +- import_playbook: certificates-backup.yml -- include: certificates.yml +- import_playbook: certificates.yml vars: etcd_certificates_redeploy: true diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml index fac8e3f02..3ef043ec8 100644 --- a/playbooks/openshift-etcd/private/scaleup.yml +++ b/playbooks/openshift-etcd/private/scaleup.yml @@ -32,7 +32,7 @@ until: etcd_add_check.rc == 0 - include_role: name: etcd - tasks_from: server_certificates + tasks_from: server_certificates.yml vars: etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" @@ -78,4 +78,4 @@ post_tasks: - include_role: name: openshift_master - tasks_from: update_etcd_client_urls + tasks_from: update_etcd_client_urls.yml diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml index 14c74baf3..695b53990 100644 --- a/playbooks/openshift-etcd/private/server_certificates.yml +++ b/playbooks/openshift-etcd/private/server_certificates.yml @@ -7,7 +7,7 @@ post_tasks: - include_role: name: etcd - tasks_from: server_certificates + tasks_from: server_certificates.yml vars: etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" diff --git a/playbooks/openshift-etcd/private/upgrade_backup.yml b/playbooks/openshift-etcd/private/upgrade_backup.yml new file mode 100644 index 000000000..22ed7e610 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_backup.yml @@ -0,0 +1,28 @@ +--- +- name: Backup etcd + hosts: oo_etcd_hosts_to_backup + roles: + - role: openshift_etcd_facts + post_tasks: + - include_role: + name: etcd + tasks_from: backup.yml + vars: + r_etcd_common_backup_tag: "{{ etcd_backup_tag }}" + r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" + r_etcd_common_backup_sufix_name: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" + +- name: Gate on etcd backup + hosts: localhost + connection: local + become: no + tasks: + - set_fact: + etcd_backup_completed: "{{ hostvars + | oo_select_keys(groups.oo_etcd_hosts_to_backup) + | oo_collect('inventory_hostname', {'r_etcd_common_backup_complete': true}) }}" + - set_fact: + etcd_backup_failed: "{{ groups.oo_etcd_hosts_to_backup | difference(etcd_backup_completed) | list }}" + - fail: + msg: "Upgrade cannot continue. The following hosts did not complete etcd backup: {{ etcd_backup_failed | join(',') }}" + when: etcd_backup_failed | length > 0 diff --git a/playbooks/openshift-etcd/private/upgrade_image_members.yml b/playbooks/openshift-etcd/private/upgrade_image_members.yml new file mode 100644 index 000000000..c133c0201 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_image_members.yml @@ -0,0 +1,17 @@ +--- +# INPUT etcd_upgrade_version +# INPUT etcd_container_version +# INPUT openshift.common.is_containerized +- name: Upgrade containerized hosts to {{ etcd_upgrade_version }} + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - include_role: + name: etcd + tasks_from: upgrade_image.yml + vars: + r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" + etcd_peer: "{{ openshift.common.hostname }}" + when: + - etcd_container_version | default('99') | version_compare(etcd_upgrade_version,'<') + - openshift.common.is_containerized | bool diff --git a/playbooks/openshift-etcd/private/upgrade_main.yml b/playbooks/openshift-etcd/private/upgrade_main.yml new file mode 100644 index 000000000..e373a4a4c --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_main.yml @@ -0,0 +1,29 @@ +--- +# For 1.4/3.4 we want to upgrade everyone to etcd-3.0. etcd docs say to +# upgrade from 2.0.x to 2.1.x to 2.2.x to 2.3.x to 3.0.x. While this is a tedius +# task for RHEL and CENTOS it's simply not possible in Fedora unless you've +# mirrored packages on your own because only the GA and latest versions are +# available in the repos. So for Fedora we'll simply skip this, sorry. + +- name: Backup etcd before upgrading anything + import_playbook: upgrade_backup.yml + vars: + etcd_backup_tag: "pre-upgrade-" + when: openshift_etcd_backup | default(true) | bool + +- name: Drop etcdctl profiles + hosts: oo_etcd_hosts_to_upgrade + tasks: + - include_role: + name: etcd + tasks_from: drop_etcdctl.yml + +- name: Perform etcd upgrade + import_playbook: upgrade_step.yml + when: openshift_etcd_upgrade | default(true) | bool + +- name: Backup etcd + import_playbook: upgrade_backup.yml + vars: + etcd_backup_tag: "post-3.0-" + when: openshift_etcd_backup | default(true) | bool diff --git a/playbooks/openshift-etcd/private/upgrade_rpm_members.yml b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml new file mode 100644 index 000000000..902c39d9c --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml @@ -0,0 +1,18 @@ +--- +# INPUT etcd_upgrade_version +# INPUT etcd_rpm_version +# INPUT openshift.common.is_containerized +- name: Upgrade to {{ etcd_upgrade_version }} + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - include_role: + name: etcd + tasks_from: upgrade_rpm.yml + vars: + r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" + etcd_peer: "{{ openshift.common.hostname }}" + when: + - etcd_rpm_version.stdout | default('99') | version_compare(etcd_upgrade_version, '<') + - ansible_distribution == 'RedHat' + - not openshift.common.is_containerized | bool diff --git a/playbooks/openshift-etcd/private/upgrade_step.yml b/playbooks/openshift-etcd/private/upgrade_step.yml new file mode 100644 index 000000000..60127fc68 --- /dev/null +++ b/playbooks/openshift-etcd/private/upgrade_step.yml @@ -0,0 +1,64 @@ +--- +- name: Determine etcd version + hosts: oo_etcd_hosts_to_upgrade + tasks: + - include_role: + name: etcd + tasks_from: version_detect.yml + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.1' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.2' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '2.2.5' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '2.3' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '2.3.7' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.0' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.0.15' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.1' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.1.3' + +- import_playbook: upgrade_rpm_members.yml + vars: + etcd_upgrade_version: '3.2' + +- import_playbook: upgrade_image_members.yml + vars: + etcd_upgrade_version: '3.2.7' + +- name: Upgrade fedora to latest + hosts: oo_etcd_hosts_to_upgrade + serial: 1 + tasks: + - include_role: + name: etcd + tasks_from: upgrade_image.yml + vars: + etcd_peer: "{{ openshift.common.hostname }}" + when: + - ansible_distribution == 'Fedora' + - not openshift.common.is_containerized | bool diff --git a/playbooks/openshift-etcd/redeploy-ca.yml b/playbooks/openshift-etcd/redeploy-ca.yml index b1d23675d..769d694ba 100644 --- a/playbooks/openshift-etcd/redeploy-ca.yml +++ b/playbooks/openshift-etcd/redeploy-ca.yml @@ -1,4 +1,4 @@ --- -- include: ../init/main.yml +- import_playbook: ../init/main.yml -- include: private/redeploy-ca.yml +- import_playbook: private/redeploy-ca.yml diff --git a/playbooks/openshift-etcd/redeploy-certificates.yml b/playbooks/openshift-etcd/redeploy-certificates.yml index 1bd302c03..753878d70 100644 --- a/playbooks/openshift-etcd/redeploy-certificates.yml +++ b/playbooks/openshift-etcd/redeploy-certificates.yml @@ -1,10 +1,10 @@ --- -- include: ../init/main.yml +- import_playbook: ../init/main.yml -- include: private/redeploy-certificates.yml +- import_playbook: private/redeploy-certificates.yml -- include: private/restart.yml +- import_playbook: private/restart.yml vars: g_etcd_certificates_expired: "{{ ('expired' in (hostvars | oo_select_keys(groups['etcd']) | oo_collect('check_results.check_results.etcd') | oo_collect('health'))) | bool }}" -- include: ../openshift-master/private/restart.yml +- import_playbook: ../openshift-master/private/restart.yml diff --git a/playbooks/openshift-etcd/upgrade.yml b/playbooks/openshift-etcd/upgrade.yml new file mode 100644 index 000000000..ccc797527 --- /dev/null +++ b/playbooks/openshift-etcd/upgrade.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/evaluate_groups.yml + +- import_playbook: private/upgrade_main.yml -- cgit v1.2.1