diff options
88 files changed, 487 insertions, 634 deletions
diff --git a/playbooks/byo/config.yml b/playbooks/byo/config.yml index 7d03914a2..4b74e5bce 100644 --- a/playbooks/byo/config.yml +++ b/playbooks/byo/config.yml @@ -1,2 +1,3 @@ --- -- include: openshift-cluster/config.yml +# TODO (rteague): Temporarily leaving this playbook to allow CI tests to operate until CI jobs are updated. +- import_playbook: ../deploy_cluster.yml diff --git a/playbooks/byo/openshift-cluster/config.yml b/playbooks/byo/openshift-cluster/config.yml deleted file mode 100644 index 57823847b..000000000 --- a/playbooks/byo/openshift-cluster/config.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../init/main.yml - -- include: ../../common/openshift-cluster/config.yml diff --git a/playbooks/byo/openshift-cluster/upgrades/upgrade_etcd.yml b/playbooks/byo/openshift-cluster/upgrades/upgrade_etcd.yml deleted file mode 100644 index a9be8dec4..000000000 --- a/playbooks/byo/openshift-cluster/upgrades/upgrade_etcd.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- include: ../../../init/evaluate_groups.yml - -- include: ../../../common/openshift-cluster/upgrades/etcd/main.yml diff --git a/playbooks/common/openshift-cluster/config.yml b/playbooks/common/openshift-cluster/config.yml deleted file mode 100644 index a8ca5e686..000000000 --- a/playbooks/common/openshift-cluster/config.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -- include: ../../openshift-checks/private/install.yml - -- include: ../../openshift-etcd/private/config.yml - -- include: ../../openshift-nfs/private/config.yml - when: groups.oo_nfs_to_config | default([]) | count > 0 - -- include: ../../openshift-loadbalancer/private/config.yml - when: groups.oo_lb_to_config | default([]) | count > 0 - -- include: ../../openshift-master/private/config.yml - -- include: ../../openshift-master/private/additional_config.yml - -- include: ../../openshift-node/private/config.yml - -- include: ../../openshift-glusterfs/private/config.yml - when: groups.oo_glusterfs_to_config | default([]) | count > 0 - -- include: ../../openshift-hosted/private/config.yml - -- include: ../../openshift-metrics/private/config.yml - when: openshift_metrics_install_metrics | default(false) | bool - -- include: ../../openshift-logging/private/config.yml - when: openshift_logging_install_logging | default(false) | bool - -- include: ../../openshift-prometheus/private/config.yml - when: openshift_hosted_prometheus_deploy | default(false) | bool - -- include: ../../openshift-service-catalog/private/config.yml - when: openshift_enable_service_catalog | default(true) | bool - -- include: ../../openshift-management/private/config.yml - when: openshift_management_install_management | default(false) | bool - -- name: Print deprecated variable warning message if necessary - hosts: oo_first_master - gather_facts: no - tasks: - - debug: msg="{{__deprecation_message}}" - when: - - __deprecation_message | default ('') | length > 0 diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins deleted file mode 120000 index 27ddaa18b..000000000 --- a/playbooks/common/openshift-cluster/upgrades/etcd/filter_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../../../filter_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins b/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins deleted file mode 120000 index cf407f69b..000000000 --- a/playbooks/common/openshift-cluster/upgrades/etcd/lookup_plugins +++ /dev/null @@ -1 +0,0 @@ -../../../../../lookup_plugins
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/roles b/playbooks/common/openshift-cluster/upgrades/etcd/roles deleted file mode 120000 index 6bc1a7aef..000000000 --- a/playbooks/common/openshift-cluster/upgrades/etcd/roles +++ /dev/null @@ -1 +0,0 @@ -../../../../../roles
\ No newline at end of file diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml index 1d59dda32..4aa63d757 100644 --- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml +++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml @@ -16,8 +16,8 @@ local_facts: embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" -- name: Upgrade and backup etcd - include: ./etcd/main.yml +- name: Backup and upgrade etcd + include: ../../../openshift-etcd/private/upgrade_main.yml # Create service signer cert when missing. Service signer certificate # is added to master config in the master_config_upgrade hook. diff --git a/playbooks/container-runtime/config.yml b/playbooks/container-runtime/config.yml new file mode 100644 index 000000000..f15aa771f --- /dev/null +++ b/playbooks/container-runtime/config.yml @@ -0,0 +1,6 @@ +--- +- import_playbook: ../init/main.yml + vars: + skip_verison: True + +- import_playbook: private/config.yml diff --git a/playbooks/container-runtime/private/config.yml b/playbooks/container-runtime/private/config.yml new file mode 100644 index 000000000..67445edeb --- /dev/null +++ b/playbooks/container-runtime/private/config.yml @@ -0,0 +1,28 @@ +--- +- hosts: "{{ l_containerized_host_groups }}" + vars: + l_chg_temp: "{{ openshift_containerized_host_groups | default([]) }}" + l_containerized_host_groups: "{{ (['oo_nodes_to_config'] | union(l_chg_temp)) | join(':') }}" + # role: container_runtime is necessary here to bring role default variables + # into the play scope. + roles: + - role: container_runtime + tasks: + - include_role: + name: container_runtime + tasks_from: package_docker.yml + when: + - not openshift_docker_use_system_container | bool + - not openshift_use_crio_only | bool + - include_role: + name: container_runtime + tasks_from: systemcontainer_docker.yml + when: + - openshift_docker_use_system_container | bool + - not openshift_use_crio_only | bool + - include_role: + name: container_runtime + tasks_from: systemcontainer_crio.yml + when: + - openshift_use_crio | bool + - openshift_docker_is_node_or_master | bool diff --git a/playbooks/container-runtime/private/roles b/playbooks/container-runtime/private/roles new file mode 120000 index 000000000..148b13206 --- /dev/null +++ b/playbooks/container-runtime/private/roles @@ -0,0 +1 @@ +../../roles/
\ No newline at end of file diff --git a/playbooks/deploy_cluster.yml b/playbooks/deploy_cluster.yml new file mode 100644 index 000000000..0e6bde09a --- /dev/null +++ b/playbooks/deploy_cluster.yml @@ -0,0 +1,46 @@ +--- +- import_playbook: init/main.yml + +- import_playbook: openshift-checks/private/install.yml + +- import_playbook: openshift-etcd/private/config.yml + +- import_playbook: openshift-nfs/private/config.yml + when: groups.oo_nfs_to_config | default([]) | count > 0 + +- import_playbook: openshift-loadbalancer/private/config.yml + when: groups.oo_lb_to_config | default([]) | count > 0 + +- import_playbook: openshift-master/private/config.yml + +- import_playbook: openshift-master/private/additional_config.yml + +- import_playbook: openshift-node/private/config.yml + +- import_playbook: openshift-glusterfs/private/config.yml + when: groups.oo_glusterfs_to_config | default([]) | count > 0 + +- import_playbook: openshift-hosted/private/config.yml + +- import_playbook: openshift-metrics/private/config.yml + when: openshift_metrics_install_metrics | default(false) | bool + +- import_playbook: openshift-logging/private/config.yml + when: openshift_logging_install_logging | default(false) | bool + +- import_playbook: openshift-prometheus/private/config.yml + when: openshift_hosted_prometheus_deploy | default(false) | bool + +- import_playbook: openshift-service-catalog/private/config.yml + when: openshift_enable_service_catalog | default(true) | bool + +- import_playbook: openshift-management/private/config.yml + when: openshift_management_install_management | default(false) | bool + +- name: Print deprecated variable warning message if necessary + hosts: oo_first_master + gather_facts: no + tasks: + - debug: msg="{{__deprecation_message}}" + when: + - __deprecation_message | default ('') | length > 0 diff --git a/playbooks/gcp/openshift-cluster/provision.yml b/playbooks/gcp/provision.yml index 097717607..9887f09f2 100644 --- a/playbooks/gcp/openshift-cluster/provision.yml +++ b/playbooks/gcp/provision.yml @@ -9,8 +9,5 @@ include_role: name: openshift_gcp -- name: run the init - include: ../../init/main.yml - -- name: run the config - include: ../../common/openshift-cluster/config.yml +- name: run the cluster deploy + include: ../deploy_cluster.yml diff --git a/playbooks/openshift-etcd/private/ca.yml b/playbooks/openshift-etcd/private/ca.yml index c9f186e72..f3bb3c2d1 100644 --- a/playbooks/openshift-etcd/private/ca.yml +++ b/playbooks/openshift-etcd/private/ca.yml @@ -7,7 +7,7 @@ tasks: - include_role: name: etcd - tasks_from: ca + tasks_from: ca.yml vars: etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" diff --git a/playbooks/openshift-etcd/private/certificates-backup.yml b/playbooks/openshift-etcd/private/certificates-backup.yml index d738c8207..ce21a1f96 100644 --- a/playbooks/openshift-etcd/private/certificates-backup.yml +++ b/playbooks/openshift-etcd/private/certificates-backup.yml @@ -5,10 +5,10 @@ tasks: - include_role: name: etcd - tasks_from: backup_generated_certificates + tasks_from: backup_generated_certificates.yml - include_role: name: etcd - tasks_from: remove_generated_certificates + tasks_from: remove_generated_certificates.yml - name: Backup deployed etcd certificates hosts: oo_etcd_to_config @@ -16,4 +16,4 @@ tasks: - include_role: name: etcd - tasks_from: backup_server_certificates + tasks_from: backup_server_certificates.yml diff --git a/playbooks/openshift-etcd/private/embedded2external.yml b/playbooks/openshift-etcd/private/embedded2external.yml index 514319b88..c7a532622 100644 --- a/playbooks/openshift-etcd/private/embedded2external.yml +++ b/playbooks/openshift-etcd/private/embedded2external.yml @@ -20,7 +20,7 @@ - name: Check the master API is ready include_role: name: openshift_master - tasks_from: check_master_api_is_ready + tasks_from: check_master_api_is_ready.yml - set_fact: master_service: "{{ openshift.common.service_type + '-master' }}" embedded_etcd_backup_suffix: "{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}" @@ -34,7 +34,7 @@ # Can't use with_items with include_role: https://github.com/ansible/ansible/issues/21285 - include_role: name: etcd - tasks_from: backup + tasks_from: backup.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_embedded_etcd: "{{ true }}" @@ -42,7 +42,7 @@ - include_role: name: etcd - tasks_from: backup.archive + tasks_from: backup.archive.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_embedded_etcd: "{{ true }}" @@ -58,7 +58,7 @@ tasks: - include_role: name: etcd - tasks_from: backup_master_etcd_certificates + tasks_from: backup_master_etcd_certificates.yml - name: Redeploy master etcd certificates import_playbook: master_etcd_certificates.yml @@ -75,10 +75,10 @@ pre_tasks: - include_role: name: etcd - tasks_from: disable_etcd + tasks_from: disable_etcd.yml - include_role: name: etcd - tasks_from: clean_data + tasks_from: clean_data.yml # 6. copy the embedded etcd backup to the external host # TODO(jchaloup): if the etcd and first master are on the same host, just copy the directory @@ -93,7 +93,7 @@ - include_role: name: etcd - tasks_from: backup.fetch + tasks_from: backup.fetch.yml vars: etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" r_etcd_common_backup_tag: pre-migrate @@ -103,7 +103,7 @@ - include_role: name: etcd - tasks_from: backup.copy + tasks_from: backup.copy.yml vars: etcd_backup_sync_directory: "{{ g_etcd_client_mktemp.stdout }}" r_etcd_common_backup_tag: pre-migrate @@ -124,14 +124,14 @@ tasks: - include_role: name: etcd - tasks_from: backup.unarchive + tasks_from: backup.unarchive.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" - include_role: name: etcd - tasks_from: backup.force_new_cluster + tasks_from: backup.force_new_cluster.yml vars: r_etcd_common_backup_tag: pre-migrate r_etcd_common_backup_sufix_name: "{{ hostvars[groups.oo_first_master.0].embedded_etcd_backup_suffix }}" @@ -145,7 +145,7 @@ tasks: - include_role: name: openshift_master - tasks_from: configure_external_etcd + tasks_from: configure_external_etcd.yml vars: etcd_peer_url_scheme: "https" etcd_ip: "{{ hostvars[groups.oo_etcd_to_config.0].openshift.common.ip }}" diff --git a/playbooks/openshift-etcd/private/migrate.yml b/playbooks/openshift-etcd/private/migrate.yml index 4269918c2..834bd242d 100644 --- a/playbooks/openshift-etcd/private/migrate.yml +++ b/playbooks/openshift-etcd/private/migrate.yml @@ -17,7 +17,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.pre_check + tasks_from: migrate.pre_check.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ ansible_default_ipv4.address }}" @@ -46,7 +46,7 @@ post_tasks: - include_role: name: etcd - tasks_from: backup + tasks_from: backup.yml vars: r_etcd_common_backup_tag: pre-migration r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" @@ -74,7 +74,7 @@ pre_tasks: - include_role: name: etcd - tasks_from: disable_etcd + tasks_from: disable_etcd.yml - name: Migrate data on first etcd hosts: oo_etcd_to_migrate[0] @@ -82,7 +82,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate + tasks_from: migrate.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ openshift.common.ip }}" @@ -95,7 +95,7 @@ tasks: - include_role: name: etcd - tasks_from: clean_data + tasks_from: clean_data.yml vars: r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" etcd_peer: "{{ openshift.common.ip }}" @@ -132,7 +132,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.add_ttls + tasks_from: migrate.add_ttls.yml vars: etcd_peer: "{{ hostvars[groups.oo_etcd_to_migrate.0].openshift.common.ip }}" etcd_url_scheme: "https" @@ -144,7 +144,7 @@ tasks: - include_role: name: etcd - tasks_from: migrate.configure_master + tasks_from: migrate.configure_master.yml when: etcd_migration_failed | length == 0 - debug: msg: "Skipping master re-configuration since migration failed." diff --git a/playbooks/openshift-etcd/private/redeploy-ca.yml b/playbooks/openshift-etcd/private/redeploy-ca.yml index cc5d57031..158bcb849 100644 --- a/playbooks/openshift-etcd/private/redeploy-ca.yml +++ b/playbooks/openshift-etcd/private/redeploy-ca.yml @@ -16,12 +16,12 @@ tasks: - include_role: name: etcd - tasks_from: backup_ca_certificates + tasks_from: backup_ca_certificates.yml - include_role: name: etcd - tasks_from: remove_ca_certificates + tasks_from: remove_ca_certificates.yml -- include: ca.yml +- import_playbook: ca.yml - name: Create temp directory for syncing certs hosts: localhost @@ -44,7 +44,7 @@ etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" -- include: restart.yml +- import_playbook: restart.yml # Do not restart etcd when etcd certificates were previously expired. when: ('expired' not in (hostvars | oo_select_keys(groups['etcd']) @@ -56,7 +56,7 @@ tasks: - include_role: name: etcd - tasks_from: retrieve_ca_certificates + tasks_from: retrieve_ca_certificates.yml vars: etcd_sync_cert_dir: "{{ hostvars['localhost'].g_etcd_mktemp.stdout }}" @@ -82,7 +82,7 @@ state: absent changed_when: false -- include: ../../openshift-master/private/restart.yml +- import_playbook: ../../openshift-master/private/restart.yml # Do not restart masters when master or etcd certificates were previously expired. when: # masters diff --git a/playbooks/openshift-etcd/private/redeploy-certificates.yml b/playbooks/openshift-etcd/private/redeploy-certificates.yml index cc1e6adf5..1c8eb27ac 100644 --- a/playbooks/openshift-etcd/private/redeploy-certificates.yml +++ b/playbooks/openshift-etcd/private/redeploy-certificates.yml @@ -11,8 +11,8 @@ # certificates were previously expired. - role: openshift_certificate_expiry -- include: certificates-backup.yml +- import_playbook: certificates-backup.yml -- include: certificates.yml +- import_playbook: certificates.yml vars: etcd_certificates_redeploy: true diff --git a/playbooks/openshift-etcd/private/scaleup.yml b/playbooks/openshift-etcd/private/scaleup.yml index fac8e3f02..3ef043ec8 100644 --- a/playbooks/openshift-etcd/private/scaleup.yml +++ b/playbooks/openshift-etcd/private/scaleup.yml @@ -32,7 +32,7 @@ until: etcd_add_check.rc == 0 - include_role: name: etcd - tasks_from: server_certificates + tasks_from: server_certificates.yml vars: etcd_peers: "{{ groups.oo_new_etcd_to_config | default([], true) }}" etcd_certificates_etcd_hosts: "{{ groups.oo_new_etcd_to_config | default([], true) }}" @@ -78,4 +78,4 @@ post_tasks: - include_role: name: openshift_master - tasks_from: update_etcd_client_urls + tasks_from: update_etcd_client_urls.yml diff --git a/playbooks/openshift-etcd/private/server_certificates.yml b/playbooks/openshift-etcd/private/server_certificates.yml index 14c74baf3..695b53990 100644 --- a/playbooks/openshift-etcd/private/server_certificates.yml +++ b/playbooks/openshift-etcd/private/server_certificates.yml @@ -7,7 +7,7 @@ post_tasks: - include_role: name: etcd - tasks_from: server_certificates + tasks_from: server_certificates.yml vars: etcd_ca_host: "{{ groups.oo_etcd_to_config.0 }}" etcd_peers: "{{ groups.oo_etcd_to_config | default([], true) }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml b/playbooks/openshift-etcd/private/upgrade_backup.yml index 531175c85..22ed7e610 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/backup.yml +++ b/playbooks/openshift-etcd/private/upgrade_backup.yml @@ -6,7 +6,7 @@ post_tasks: - include_role: name: etcd - tasks_from: backup + tasks_from: backup.yml vars: r_etcd_common_backup_tag: "{{ etcd_backup_tag }}" r_etcd_common_embedded_etcd: "{{ groups.oo_etcd_to_config | default([]) | length == 0 }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_image_members.yml b/playbooks/openshift-etcd/private/upgrade_image_members.yml index 6fca42bd0..c133c0201 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_image_members.yml +++ b/playbooks/openshift-etcd/private/upgrade_image_members.yml @@ -8,7 +8,7 @@ tasks: - include_role: name: etcd - tasks_from: upgrade_image + tasks_from: upgrade_image.yml vars: r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" etcd_peer: "{{ openshift.common.hostname }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml b/playbooks/openshift-etcd/private/upgrade_main.yml index 5b8ba3bb2..e373a4a4c 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/main.yml +++ b/playbooks/openshift-etcd/private/upgrade_main.yml @@ -6,7 +6,7 @@ # available in the repos. So for Fedora we'll simply skip this, sorry. - name: Backup etcd before upgrading anything - include: backup.yml + import_playbook: upgrade_backup.yml vars: etcd_backup_tag: "pre-upgrade-" when: openshift_etcd_backup | default(true) | bool @@ -16,14 +16,14 @@ tasks: - include_role: name: etcd - tasks_from: drop_etcdctl + tasks_from: drop_etcdctl.yml - name: Perform etcd upgrade - include: ./upgrade.yml + import_playbook: upgrade_step.yml when: openshift_etcd_upgrade | default(true) | bool - name: Backup etcd - include: backup.yml + import_playbook: upgrade_backup.yml vars: etcd_backup_tag: "post-3.0-" when: openshift_etcd_backup | default(true) | bool diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_rpm_members.yml b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml index 51e8786b3..902c39d9c 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade_rpm_members.yml +++ b/playbooks/openshift-etcd/private/upgrade_rpm_members.yml @@ -8,7 +8,7 @@ tasks: - include_role: name: etcd - tasks_from: upgrade_rpm + tasks_from: upgrade_rpm.yml vars: r_etcd_upgrade_version: "{{ etcd_upgrade_version }}" etcd_peer: "{{ openshift.common.hostname }}" diff --git a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml b/playbooks/openshift-etcd/private/upgrade_step.yml index c5ff4133c..60127fc68 100644 --- a/playbooks/common/openshift-cluster/upgrades/etcd/upgrade.yml +++ b/playbooks/openshift-etcd/private/upgrade_step.yml @@ -6,47 +6,47 @@ name: etcd tasks_from: version_detect.yml -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '2.1' -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '2.2' -- include: upgrade_image_members.yml +- import_playbook: upgrade_image_members.yml vars: etcd_upgrade_version: '2.2.5' -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '2.3' -- include: upgrade_image_members.yml +- import_playbook: upgrade_image_members.yml vars: etcd_upgrade_version: '2.3.7' -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '3.0' -- include: upgrade_image_members.yml +- import_playbook: upgrade_image_members.yml vars: etcd_upgrade_version: '3.0.15' -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '3.1' -- include: upgrade_image_members.yml +- import_playbook: upgrade_image_members.yml vars: etcd_upgrade_version: '3.1.3' -- include: upgrade_rpm_members.yml +- import_playbook: upgrade_rpm_members.yml vars: etcd_upgrade_version: '3.2' -- include: upgrade_image_members.yml +- import_playbook: upgrade_image_members.yml vars: etcd_upgrade_version: '3.2.7' @@ -56,7 +56,7 @@ tasks: - include_role: name: etcd - tasks_from: upgrade_image + tasks_from: upgrade_image.yml vars: etcd_peer: "{{ openshift.common.hostname }}" when: diff --git a/playbooks/openshift-etcd/redeploy-ca.yml b/playbooks/openshift-etcd/redeploy-ca.yml index b1d23675d..769d694ba 100644 --- a/playbooks/openshift-etcd/redeploy-ca.yml +++ b/playbooks/openshift-etcd/redeploy-ca.yml @@ -1,4 +1,4 @@ --- -- include: ../init/main.yml +- import_playbook: ../init/main.yml -- include: private/redeploy-ca.yml +- import_playbook: private/redeploy-ca.yml diff --git a/playbooks/openshift-etcd/redeploy-certificates.yml b/playbooks/openshift-etcd/redeploy-certificates.yml index 1bd302c03..753878d70 100644 --- a/playbooks/openshift-etcd/redeploy-certificates.yml +++ b/playbooks/openshift-etcd/redeploy-certificates.yml @@ -1,10 +1,10 @@ --- -- include: ../init/main.yml +- import_playbook: ../init/main.yml -- include: private/redeploy-certificates.yml +- import_playbook: private/redeploy-certificates.yml -- include: private/restart.yml +- import_playbook: private/restart.yml vars: g_etcd_certificates_expired: "{{ ('expired' in (hostvars | oo_select_keys(groups['etcd']) | oo_collect('check_results.check_results.etcd') | oo_collect('health'))) | bool }}" -- include: ../openshift-master/private/restart.yml +- import_playbook: ../openshift-master/private/restart.yml diff --git a/playbooks/openshift-etcd/upgrade.yml b/playbooks/openshift-etcd/upgrade.yml new file mode 100644 index 000000000..ccc797527 --- /dev/null +++ b/playbooks/openshift-etcd/upgrade.yml @@ -0,0 +1,4 @@ +--- +- import_playbook: ../init/evaluate_groups.yml + +- import_playbook: private/upgrade_main.yml diff --git a/playbooks/openstack/openshift-cluster/install.yml b/playbooks/openstack/openshift-cluster/install.yml index 1c4f609e3..8ed01b192 100644 --- a/playbooks/openstack/openshift-cluster/install.yml +++ b/playbooks/openstack/openshift-cluster/install.yml @@ -8,8 +8,5 @@ # values here. We do it in the OSEv3 group vars. Do we need to add # some logic here? -- name: run the initialization - include: ../../init/main.yml - -- name: run the config - include: ../../common/openshift-cluster/config.yml +- name: run the cluster deploy + include: ../../deploy_cluster.yml diff --git a/playbooks/openstack/openshift-cluster/provision.yml b/playbooks/openstack/openshift-cluster/provision.yml index 9d75e515a..3e295b2c8 100644 --- a/playbooks/openstack/openshift-cluster/provision.yml +++ b/playbooks/openstack/openshift-cluster/provision.yml @@ -26,6 +26,9 @@ - name: Gather facts for the new nodes setup: +- name: set common facts + include: ../../init/facts.yml + # TODO(shadower): consider splitting this up so people can stop here # and configure their DNS if they have to. @@ -44,6 +47,13 @@ hosts: oo_all_hosts become: yes gather_facts: yes + roles: + - role: rhel_subscribe + when: + - ansible_distribution == "RedHat" + - rhsub_user | default(False) + - rhsub_pass | default(False) + tasks: - name: Install dependencies include_role: diff --git a/playbooks/openstack/sample-inventory/group_vars/all.yml b/playbooks/openstack/sample-inventory/group_vars/all.yml index ae1528123..c7afe9a24 100644 --- a/playbooks/openstack/sample-inventory/group_vars/all.yml +++ b/playbooks/openstack/sample-inventory/group_vars/all.yml @@ -82,27 +82,10 @@ openshift_openstack_docker_volume_size: "15" openshift_openstack_subnet_prefix: "192.168.99" -## Red Hat subscription defaults to false which means we will not attempt to -## subscribe the nodes -#rhsm_register: False - -# # Using Red Hat Satellite: -#rhsm_register: True -#rhsm_satellite: 'sat-6.example.com' -#rhsm_org: 'OPENSHIFT_ORG' -#rhsm_activationkey: '<activation-key>' - -# # Or using RHN username, password and optionally pool: -#rhsm_register: True -#rhsm_username: '<username>' -#rhsm_password: '<password>' -#rhsm_pool: '<pool id>' - -#rhsm_repos: -# - "rhel-7-server-rpms" -# - "rhel-7-server-ose-3.5-rpms" -# - "rhel-7-server-extras-rpms" -# - "rhel-7-fast-datapath-rpms" +## Red Hat subscription: +#rhsub_user: '<username>' +#rhsub_pass: '<password>' +#rhsub_pool: '<pool name>' # # Roll-your-own DNS diff --git a/playbooks/prerequisites.yml b/playbooks/prerequisites.yml index 7dd59c5d8..0cc5fcef8 100644 --- a/playbooks/prerequisites.yml +++ b/playbooks/prerequisites.yml @@ -1,12 +1,6 @@ --- -- include: init/main.yml +- import_playbook: init/main.yml vars: skip_verison: True -- hosts: "{{ l_containerized_host_groups }}" - vars: - l_chg_temp: "{{ openshift_containerized_host_groups | default([]) }}" - l_containerized_host_groups: "{{ (['oo_nodes_to_config'] | union(l_chg_temp)) | join(':') }}" - tasks: - - include_role: - name: container_runtime +- import_playbook: container-runtime/private/config.yml diff --git a/roles/container_runtime/README.md b/roles/container_runtime/README.md index e363c1714..51f469aaf 100644 --- a/roles/container_runtime/README.md +++ b/roles/container_runtime/README.md @@ -1,18 +1,23 @@ -Docker +Container Runtime ========= Ensures docker package or system container is installed, and optionally raises timeout for systemd-udevd.service to 5 minutes. container-daemon.json items may be found at https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file -Requirements ------------- +This role is designed to be used with include_role and tasks_from. -Ansible 2.2 +Entry points +------------ +* package_docker.yml - install and setup docker container runtime. +* systemcontainer_docker.yml - utilize docker + systemcontainer +* systemcontainer_crio.yml - utilize crio + systemcontainer +* registry_auth.yml - place docker login credentials. -Mandator Role Variables --------------- +Requirements +------------ +Ansible 2.4 Dependencies @@ -24,9 +29,9 @@ Example Playbook ---------------- - hosts: servers - roles: - - role: container_runtime - docker_udev_workaround: "true" + tasks: + - include_role: container_runtime + tasks_from: package_docker.yml License ------- @@ -36,4 +41,4 @@ ASL 2.0 Author Information ------------------ -OpenShift operations, Red Hat, Inc +Red Hat, Inc diff --git a/roles/container_runtime/defaults/main.yml b/roles/container_runtime/defaults/main.yml index 62b3e141a..bd96965ac 100644 --- a/roles/container_runtime/defaults/main.yml +++ b/roles/container_runtime/defaults/main.yml @@ -71,10 +71,59 @@ docker_no_proxy: "{{ openshift.common.no_proxy | default('') }}" openshift_use_crio: False openshift_use_crio_only: False +l_openshift_image_tag_default: "{{ openshift_release | default('latest') }}" +l_openshift_image_tag: "{{ openshift_image_tag | default(l_openshift_image_tag_default) | string}}" +# --------------------- # +# systemcontainers_crio # +# --------------------- # l_insecure_crio_registries: "{{ '\"{}\"'.format('\", \"'.join(l2_docker_insecure_registries)) }}" l_crio_registries: "{{ l2_docker_additional_registries + ['docker.io'] }}" l_additional_crio_registries: "{{ '\"{}\"'.format('\", \"'.join(l_crio_registries)) }}" -l_openshift_image_tag_default: "{{ openshift_release }}" -l_openshift_image_tag: "{{ openshift_image_tag | default(l_openshift_image_tag_default) | string}}" +openshift_crio_image_tag_default: "latest" + +l_crt_crio_image_tag_dict: + openshift-enterprise: "{{ l_openshift_image_tag }}" + origin: "{{ openshift_crio_image_tag | default(openshift_crio_image_tag_default) }}" + +l_crt_crio_image_prepend_dict: + openshift-enterprise: "registry.access.redhat.com/openshift3" + origin: "docker.io/gscrivano" + +l_crt_crio_image_dict: + Fedora: + crio_image_name: "cri-o-fedora" + crio_image_tag: "latest" + CentOS: + crio_image_name: "cri-o-centos" + crio_image_tag: "latest" + RedHat: + crio_image_name: "cri-o" + crio_image_tag: "{{ openshift_crio_image_tag | default(l_crt_crio_image_tag_dict[openshift_deployment_type]) }}" + +l_crio_image_prepend: "{{ l_crt_crio_image_prepend_dict[openshift_deployment_type] }}" +l_crio_image_name: "{{ l_crt_crio_image_dict[ansible_distribution]['crio_image_name'] }}" +l_crio_image_tag: "{{ l_crt_crio_image_dict[ansible_distribution] }}" + +l_crio_image_default: "{{ l_crio_image_prepend }}/{{ l_crio_image_name }}:{{ l_crio_image_tag }}" +l_crio_image: "{{ openshift_crio_systemcontainer_image_override | default(l_crio_image_default) }}" + +# ----------------------- # +# systemcontainers_docker # +# ----------------------- # +l_crt_docker_image_prepend_dict: + Fedora: "registry.fedoraproject.org/f25" + Centos: "docker.io/gscrivano" + RedHat: "registry.access.redhat.com/openshift3" + +openshift_docker_image_tag_default: "latest" +l_crt_docker_image_tag_dict: + openshift-enterprise: "{{ l_openshift_image_tag }}" + origin: "{{ openshift_docker_image_tag | default(openshift_docker_image_tag_default) }}" + +l_docker_image_prepend: "{{ l_crt_docker_image_prepend_dict[ansible_distribution] }}" +l_docker_image_tag: "{{ l_crt_docker_image_tag_dict[openshift_deployment_type] }}" + +l_docker_image_default: "{{ l_docker_image_prepend }}/{{ openshift_docker_service_name }}:{{ l_docker_image_tag }}" +l_docker_image: "{{ openshift_docker_systemcontainer_image_override | default(l_docker_image_default) }}" diff --git a/roles/openshift_atomic/tasks/proxy.yml b/roles/container_runtime/tasks/common/atomic_proxy.yml index dde099984..dde099984 100644 --- a/roles/openshift_atomic/tasks/proxy.yml +++ b/roles/container_runtime/tasks/common/atomic_proxy.yml diff --git a/roles/container_runtime/tasks/common/post.yml b/roles/container_runtime/tasks/common/post.yml new file mode 100644 index 000000000..d790eb2c0 --- /dev/null +++ b/roles/container_runtime/tasks/common/post.yml @@ -0,0 +1,26 @@ +--- +- name: Ensure /var/lib/containers exists + file: + path: /var/lib/containers + state: directory + +- name: Fix SELinux Permissions on /var/lib/containers + command: "restorecon -R /var/lib/containers/" + changed_when: false + +- meta: flush_handlers + +# This needs to run after docker is restarted to account for proxy settings. +# registry_auth is called directly with include_role in some places, so we +# have to put it in the root of the tasks/ directory. +- include_tasks: ../registry_auth.yml + +- name: stat the docker data dir + stat: + path: "{{ docker_default_storage_path }}" + register: dockerstat + +- include_tasks: setup_docker_symlink.yml + when: + - openshift_use_crio + - dockerstat.stat.islnk is defined and not (dockerstat.stat.islnk | bool) diff --git a/roles/container_runtime/tasks/common/pre.yml b/roles/container_runtime/tasks/common/pre.yml new file mode 100644 index 000000000..990fe66da --- /dev/null +++ b/roles/container_runtime/tasks/common/pre.yml @@ -0,0 +1,12 @@ +--- +- include_tasks: udev_workaround.yml + when: docker_udev_workaround | default(False) | bool + +- name: Add enterprise registry, if necessary + set_fact: + l2_docker_additional_registries: "{{ l2_docker_additional_registries + [openshift_docker_ent_reg] }}" + when: + - openshift.common.deployment_type == 'openshift-enterprise' + - openshift_docker_ent_reg != '' + - openshift_docker_ent_reg not in l2_docker_additional_registries + - not openshift_use_crio_only | bool diff --git a/roles/container_runtime/tasks/common/setup_docker_symlink.yml b/roles/container_runtime/tasks/common/setup_docker_symlink.yml new file mode 100644 index 000000000..d7aeb192e --- /dev/null +++ b/roles/container_runtime/tasks/common/setup_docker_symlink.yml @@ -0,0 +1,38 @@ +--- +- block: + - name: stop the current running docker + systemd: + state: stopped + name: "{{ openshift_docker_service_name }}" + + - name: copy "{{ docker_default_storage_path }}" to "{{ docker_alt_storage_path }}" + command: "cp -r {{ docker_default_storage_path }} {{ docker_alt_storage_path }}" + register: results + failed_when: + - results.rc != 0 + + - name: "Set the selinux context on {{ docker_alt_storage_path }}" + command: "semanage fcontext -a -e {{ docker_default_storage_path }} {{ docker_alt_storage_path }}" + register: results + failed_when: + - results.rc == 1 + - "'already exists' not in results.stderr" + + - name: "restorecon the {{ docker_alt_storage_path }}" + command: "restorecon -r {{ docker_alt_storage_path }}" + + - name: Remove the old docker location + file: + state: absent + path: "{{ docker_default_storage_path }}" + + - name: Setup the link + file: + state: link + src: "{{ docker_alt_storage_path }}" + path: "{{ docker_default_storage_path }}" + + - name: start docker + systemd: + state: started + name: "{{ openshift_docker_service_name }}" diff --git a/roles/container_runtime/tasks/common/syscontainer_packages.yml b/roles/container_runtime/tasks/common/syscontainer_packages.yml new file mode 100644 index 000000000..715ed492d --- /dev/null +++ b/roles/container_runtime/tasks/common/syscontainer_packages.yml @@ -0,0 +1,28 @@ +--- + +- name: Ensure container-selinux is installed + package: + name: container-selinux + state: present + when: not openshift.common.is_atomic | bool + register: result + until: result | success + +# Used to pull and install the system container +- name: Ensure atomic is installed + package: + name: atomic + state: present + when: not openshift.common.is_atomic | bool + register: result + until: result | success + +# At the time of writing the atomic command requires runc for it's own use. This +# task is here in the even that the atomic package ever removes the dependency. +- name: Ensure runc is installed + package: + name: runc + state: present + when: not openshift.common.is_atomic | bool + register: result + until: result | success diff --git a/roles/container_runtime/tasks/udev_workaround.yml b/roles/container_runtime/tasks/common/udev_workaround.yml index 257c3123d..257c3123d 100644 --- a/roles/container_runtime/tasks/udev_workaround.yml +++ b/roles/container_runtime/tasks/common/udev_workaround.yml diff --git a/roles/container_runtime/tasks/docker_sanity.yml b/roles/container_runtime/tasks/docker_sanity.yml new file mode 100644 index 000000000..e62cf5505 --- /dev/null +++ b/roles/container_runtime/tasks/docker_sanity.yml @@ -0,0 +1,27 @@ +--- +# Sanity checks to ensure the role will complete and provide helpful error +# messages for common problems. + +- name: Error out if Docker pre-installed but too old + fail: + msg: "Docker {{ curr_docker_version.stdout }} is installed, but >= 1.9.1 is required." + when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.9.1', '<') and not docker_version is defined + +- name: Error out if requested Docker is too old + fail: + msg: "Docker {{ docker_version }} requested, but >= 1.9.1 is required." + when: docker_version is defined and docker_version | version_compare('1.9.1', '<') + +# If a docker_version was requested, sanity check that we can install or upgrade to it, and +# no downgrade is required. +- name: Fail if Docker version requested but downgrade is required + fail: + msg: "Docker {{ curr_docker_version.stdout }} is installed, but version {{ docker_version }} was requested." + when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and docker_version is defined and curr_docker_version.stdout | version_compare(docker_version, '>') + +# This involves an extremely slow migration process, users should instead run the +# Docker 1.10 upgrade playbook to accomplish this. +- name: Error out if attempting to upgrade Docker across the 1.10 boundary + fail: + msg: "Cannot upgrade Docker to >= 1.10, please upgrade or remove Docker manually, or use the Docker upgrade playbook if OpenShift is already installed." + when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.10', '<') and docker_version is defined and docker_version | version_compare('1.10', '>=') diff --git a/roles/container_runtime/tasks/main.yml b/roles/container_runtime/tasks/main.yml index 6d68082b1..96d8606c6 100644 --- a/roles/container_runtime/tasks/main.yml +++ b/roles/container_runtime/tasks/main.yml @@ -1,85 +1,2 @@ --- -- include_tasks: udev_workaround.yml - when: docker_udev_workaround | default(False) | bool - -- name: Add enterprise registry, if necessary - set_fact: - l2_docker_additional_registries: "{{ l2_docker_additional_registries + [openshift_docker_ent_reg] }}" - when: - - openshift.common.deployment_type == 'openshift-enterprise' - - openshift_docker_ent_reg != '' - - openshift_docker_ent_reg not in l2_docker_additional_registries - - not openshift_use_crio_only | bool - -- name: Use Package Docker if Requested - include_tasks: package_docker.yml - when: - - not openshift_docker_use_system_container - - not openshift_use_crio_only - -- name: Ensure /var/lib/containers exists - file: - path: /var/lib/containers - state: directory - -- name: Fix SELinux Permissions on /var/lib/containers - command: "restorecon -R /var/lib/containers/" - changed_when: false - -- name: Use System Container Docker if Requested - include_tasks: systemcontainer_docker.yml - when: - - openshift_docker_use_system_container - - not openshift_use_crio_only - -- name: Add CRI-O usage Requested - include_tasks: systemcontainer_crio.yml - when: - - openshift_use_crio - - openshift_docker_is_node_or_master | bool - -- name: stat the docker data dir - stat: - path: "{{ docker_default_storage_path }}" - register: dockerstat - -- when: - - openshift_use_crio - - dockerstat.stat.islnk is defined and not (dockerstat.stat.islnk | bool) - block: - - name: stop the current running docker - systemd: - state: stopped - name: "{{ openshift_docker_service_name }}" - - - name: copy "{{ docker_default_storage_path }}" to "{{ docker_alt_storage_path }}" - command: "cp -r {{ docker_default_storage_path }} {{ docker_alt_storage_path }}" - register: results - failed_when: - - results.rc != 0 - - - name: "Set the selinux context on {{ docker_alt_storage_path }}" - command: "semanage fcontext -a -e {{ docker_default_storage_path }} {{ docker_alt_storage_path }}" - register: results - failed_when: - - results.rc == 1 - - "'already exists' not in results.stderr" - - - name: "restorecon the {{ docker_alt_storage_path }}" - command: "restorecon -r {{ docker_alt_storage_path }}" - - - name: Remove the old docker location - file: - state: absent - path: "{{ docker_default_storage_path }}" - - - name: Setup the link - file: - state: link - src: "{{ docker_alt_storage_path }}" - path: "{{ docker_default_storage_path }}" - - - name: start docker - systemd: - state: started - name: "{{ openshift_docker_service_name }}" +# This role is meant to be used with include_role and tasks_from. diff --git a/roles/container_runtime/tasks/package_docker.yml b/roles/container_runtime/tasks/package_docker.yml index 40ab75a25..89899c9cf 100644 --- a/roles/container_runtime/tasks/package_docker.yml +++ b/roles/container_runtime/tasks/package_docker.yml @@ -1,4 +1,6 @@ --- +- include_tasks: common/pre.yml + - name: Get current installed Docker version command: "{{ repoquery_installed }} --qf '%{version}' docker" when: not openshift.common.is_atomic | bool @@ -7,35 +9,16 @@ until: curr_docker_version | succeeded changed_when: false -- name: Error out if Docker pre-installed but too old - fail: - msg: "Docker {{ curr_docker_version.stdout }} is installed, but >= 1.9.1 is required." - when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.9.1', '<') and not docker_version is defined - -- name: Error out if requested Docker is too old - fail: - msg: "Docker {{ docker_version }} requested, but >= 1.9.1 is required." - when: docker_version is defined and docker_version | version_compare('1.9.1', '<') - -# If a docker_version was requested, sanity check that we can install or upgrade to it, and -# no downgrade is required. -- name: Fail if Docker version requested but downgrade is required - fail: - msg: "Docker {{ curr_docker_version.stdout }} is installed, but version {{ docker_version }} was requested." - when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and docker_version is defined and curr_docker_version.stdout | version_compare(docker_version, '>') - -# This involves an extremely slow migration process, users should instead run the -# Docker 1.10 upgrade playbook to accomplish this. -- name: Error out if attempting to upgrade Docker across the 1.10 boundary - fail: - msg: "Cannot upgrade Docker to >= 1.10, please upgrade or remove Docker manually, or use the Docker upgrade playbook if OpenShift is already installed." - when: not curr_docker_version | skipped and curr_docker_version.stdout != '' and curr_docker_version.stdout | version_compare('1.10', '<') and docker_version is defined and docker_version | version_compare('1.10', '>=') +# Some basic checks to ensure the role will complete +- include_tasks: docker_sanity.yml # Make sure Docker is installed, but does not update a running version. # Docker upgrades are handled by a separate playbook. # Note: The curr_docker_version.stdout check can be removed when https://github.com/ansible/ansible/issues/33187 gets fixed. - name: Install Docker - package: name=docker{{ '-' + docker_version if docker_version is defined else '' }} state=present + package: + name: "docker{{ '-' + docker_version if docker_version is defined else '' }}" + state: present when: not openshift.common.is_atomic | bool and not curr_docker_version | skipped and not curr_docker_version.stdout != '' register: result until: result | success @@ -161,7 +144,4 @@ - set_fact: docker_service_status_changed: "{{ (r_docker_package_docker_start_result | changed) and (r_docker_already_running_result.stdout != 'ActiveState=active' ) }}" -- meta: flush_handlers - -# This needs to run after docker is restarted to account for proxy settings. -- include_tasks: registry_auth.yml +- include_tasks: common/post.yml diff --git a/roles/container_runtime/tasks/systemcontainer_crio.yml b/roles/container_runtime/tasks/systemcontainer_crio.yml index 8dcfe60ef..5ea7df650 100644 --- a/roles/container_runtime/tasks/systemcontainer_crio.yml +++ b/roles/container_runtime/tasks/systemcontainer_crio.yml @@ -1,39 +1,14 @@ --- # TODO: Much of this file is shared with container engine tasks - -- name: Ensure container-selinux is installed - package: - name: container-selinux - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success - - name: Check we are not using node as a Docker container with CRI-O fail: msg='Cannot use CRI-O with node configured as a Docker container' when: - openshift.common.is_containerized | bool - not openshift.common.is_node_system_container | bool -# Used to pull and install the system container -- name: Ensure atomic is installed - package: - name: atomic - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success - -# At the time of writing the atomic command requires runc for it's own use. This -# task is here in the even that the atomic package ever removes the dependency. -- name: Ensure runc is installed - package: - name: runc - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success +- include_tasks: common/pre.yml +- include_tasks: common/syscontainer_packages.yml - name: Check that overlay is in the kernel shell: lsmod | grep overlay @@ -60,50 +35,11 @@ state: restarted - name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - -- block: - - - name: Set CRI-O image defaults - set_fact: - l_crio_image_prepend: "docker.io/gscrivano" - l_crio_image_name: "cri-o-fedora" - l_crio_image_tag: "latest" - - - name: Use Centos based image when distribution is CentOS - set_fact: - l_crio_image_name: "cri-o-centos" - when: ansible_distribution == "CentOS" - - - name: Set CRI-O image tag - set_fact: - l_crio_image_tag: "{{ l_openshift_image_tag }}" - when: - - openshift_deployment_type == 'openshift-enterprise' - - - name: Use RHEL based image when distribution is Red Hat - set_fact: - l_crio_image_prepend: "registry.access.redhat.com/openshift3" - l_crio_image_name: "cri-o" - when: ansible_distribution == "RedHat" - - - name: Set the full image name - set_fact: - l_crio_image: "{{ l_crio_image_prepend }}/{{ l_crio_image_name }}:{{ l_crio_image_tag }}" - - # For https://github.com/openshift/aos-cd-jobs/pull/624#pullrequestreview-61816548 - - name: Use a specific image if requested - set_fact: - l_crio_image: "{{ openshift_crio_systemcontainer_image_override }}" - when: - - openshift_crio_systemcontainer_image_override is defined - - openshift_crio_systemcontainer_image_override != "" - - # Be nice and let the user see the variable result - - debug: - var: l_crio_image + include_tasks: common/atomic_proxy.yml + +# Be nice and let the user see the variable result +- debug: + var: l_crio_image # NOTE: no_proxy added as a workaround until https://github.com/projectatomic/atomic/pull/999 is released - name: Pre-pull CRI-O System Container image @@ -112,7 +48,6 @@ environment: NO_PROXY: "{{ openshift.common.no_proxy | default('') }}" - - name: Install CRI-O System Container oc_atomic_container: name: "cri-o" @@ -139,8 +74,7 @@ state: directory - name: setup firewall for CRI-O - include_tasks: crio_firewall.yml - static: yes + import_tasks: crio_firewall.yml - name: Configure the CNI network template: @@ -155,10 +89,8 @@ daemon_reload: yes register: start_result -- meta: flush_handlers - # If we are using crio only, docker.service might not be available for # 'docker login' -- include_tasks: registry_auth.yml +- include_tasks: common/post.yml vars: openshift_docker_alternative_creds: "{{ openshift_use_crio_only }}" diff --git a/roles/container_runtime/tasks/systemcontainer_docker.yml b/roles/container_runtime/tasks/systemcontainer_docker.yml index 84217e50c..10570fe34 100644 --- a/roles/container_runtime/tasks/systemcontainer_docker.yml +++ b/roles/container_runtime/tasks/systemcontainer_docker.yml @@ -11,32 +11,9 @@ traditional docker package install. Otherwise, comment out openshift_docker_options in your inventory file. -- name: Ensure container-selinux is installed - package: - name: container-selinux - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success - -# Used to pull and install the system container -- name: Ensure atomic is installed - package: - name: atomic - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success +- include_tasks: common/pre.yml -# At the time of writing the atomic command requires runc for it's own use. This -# task is here in the even that the atomic package ever removes the dependency. -- name: Ensure runc is installed - package: - name: runc - state: present - when: not openshift.common.is_atomic | bool - register: result - until: result | success +- include_tasks: common/syscontainer_packages.yml # Make sure Docker is installed so we are able to use the client - name: Install Docker so we can use the client @@ -59,48 +36,11 @@ delay: 30 - name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - -- block: - - - name: Set to default prepend - set_fact: - l_docker_image_prepend: "gscrivano" - l_docker_image_tag: "latest" - - - name: Set container engine image tag - set_fact: - l_docker_image_tag: "{{ l_openshift_image_tag }}" - when: - - openshift_deployment_type == 'openshift-enterprise' - - - name: Use Red Hat Registry for image when distribution is Red Hat - set_fact: - l_docker_image_prepend: "registry.access.redhat.com/openshift3" - when: ansible_distribution == 'RedHat' - - - name: Use Fedora Registry for image when distribution is Fedora - set_fact: - l_docker_image_prepend: "registry.fedoraproject.org/f25" - when: ansible_distribution == 'Fedora' - - - name: Set the full image name - set_fact: - l_docker_image: "{{ l_docker_image_prepend }}/{{ openshift_docker_service_name }}:{{ l_docker_image_tag }}" - - # For https://github.com/openshift/openshift-ansible/pull/5354#issuecomment-328552959 - - name: Use a specific image if requested - set_fact: - l_docker_image: "{{ openshift_docker_systemcontainer_image_override }}" - when: - - openshift_docker_systemcontainer_image_override is defined - - openshift_docker_systemcontainer_image_override != "" - - # Be nice and let the user see the variable result - - debug: - var: l_docker_image + include_tasks: common/atomic_proxy.yml + +# Be nice and let the user see the variable result +- debug: + var: l_docker_image # NOTE: no_proxy added as a workaround until https://github.com/projectatomic/atomic/pull/999 is released - name: Pre-pull Container Engine System Container image @@ -154,10 +94,8 @@ - set_fact: docker_service_status_changed: "{{ r_docker_systemcontainer_docker_start_result | changed }}" -- meta: flush_handlers - # Since docker is running as a system container, docker login will fail to create # credentials. Use alternate method if requiring authenticated registries. -- include_tasks: registry_auth.yml +- include_tasks: common/post.yml vars: openshift_docker_alternative_creds: True diff --git a/roles/etcd/tasks/system_container.yml b/roles/etcd/tasks/system_container.yml index 82ac4fc84..ca8b6a707 100644 --- a/roles/etcd/tasks/system_container.yml +++ b/roles/etcd/tasks/system_container.yml @@ -1,9 +1,4 @@ --- -- name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - - name: Pull etcd system container command: atomic pull --storage=ostree {{ etcd_image }} register: pull_result diff --git a/roles/openshift_atomic/README.md b/roles/openshift_atomic/README.md deleted file mode 100644 index 8c10c9991..000000000 --- a/roles/openshift_atomic/README.md +++ /dev/null @@ -1,28 +0,0 @@ -OpenShift Atomic -================ - -This role houses atomic specific tasks. - -Requirements ------------- - -Role Variables --------------- - -Dependencies ------------- - -Example Playbook ----------------- - -``` -- name: Ensure atomic proxies are defined - hosts: localhost - roles: - - role: openshift_atomic -``` - -License -------- - -Apache License Version 2.0 diff --git a/roles/openshift_atomic/meta/main.yml b/roles/openshift_atomic/meta/main.yml deleted file mode 100644 index ea129f514..000000000 --- a/roles/openshift_atomic/meta/main.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -galaxy_info: - author: OpenShift - description: Atomic related tasks - company: Red Hat, Inc - license: ASL 2.0 - min_ansible_version: 2.2 - platforms: - - name: EL - versions: - - 7 -dependencies: -- role: lib_openshift diff --git a/roles/openshift_aws/tasks/build_node_group.yml b/roles/openshift_aws/tasks/build_node_group.yml index 7e8e9b679..2c1e88cfb 100644 --- a/roles/openshift_aws/tasks/build_node_group.yml +++ b/roles/openshift_aws/tasks/build_node_group.yml @@ -28,10 +28,10 @@ l_epoch_time: "{{ ansible_date_time.epoch }}" - when: openshift_aws_create_iam_role - include: iam_role.yml + include_tasks: iam_role.yml - when: openshift_aws_create_launch_config - include: launch_config.yml + include_tasks: launch_config.yml - when: openshift_aws_create_scale_group - include: scale_group.yml + include_tasks: scale_group.yml diff --git a/roles/openshift_aws/tasks/launch_config.yml b/roles/openshift_aws/tasks/launch_config.yml index 0dbeba5a0..fed80b7eb 100644 --- a/roles/openshift_aws/tasks/launch_config.yml +++ b/roles/openshift_aws/tasks/launch_config.yml @@ -9,7 +9,7 @@ when: - openshift_deployment_type is undefined -- include: launch_config_create.yml +- include_tasks: launch_config_create.yml with_dict: "{{ l_nodes_to_build }}" loop_control: loop_var: launch_config_item diff --git a/roles/openshift_aws/tasks/provision.yml b/roles/openshift_aws/tasks/provision.yml index 91538ed5c..06f649343 100644 --- a/roles/openshift_aws/tasks/provision.yml +++ b/roles/openshift_aws/tasks/provision.yml @@ -1,16 +1,16 @@ --- - when: openshift_aws_create_iam_cert | bool name: create the iam_cert for elb certificate - include: iam_cert.yml + include_tasks: iam_cert.yml - when: openshift_aws_create_s3 | bool name: create s3 bucket for registry - include: s3.yml + include_tasks: s3.yml -- include: vpc_and_subnet_id.yml +- include_tasks: vpc_and_subnet_id.yml - name: create elbs - include: elb.yml + include_tasks: elb.yml with_dict: "{{ openshift_aws_elb_dict }}" vars: l_elb_security_groups: "{{ openshift_aws_elb_security_groups }}" @@ -19,7 +19,7 @@ loop_var: l_elb_dict_item - name: include scale group creation for master - include: build_node_group.yml + include_tasks: build_node_group.yml vars: l_nodes_to_build: "{{ openshift_aws_master_group_config }}" l_launch_config_security_groups: "{{ openshift_aws_launch_config_security_groups }}" diff --git a/roles/openshift_aws/tasks/provision_instance.yml b/roles/openshift_aws/tasks/provision_instance.yml index 3349acb7a..8cc75cd0c 100644 --- a/roles/openshift_aws/tasks/provision_instance.yml +++ b/roles/openshift_aws/tasks/provision_instance.yml @@ -3,7 +3,7 @@ set_fact: openshift_node_bootstrap: True -- include: vpc_and_subnet_id.yml +- include_tasks: vpc_and_subnet_id.yml - name: create instance for ami creation ec2: diff --git a/roles/openshift_aws/tasks/provision_nodes.yml b/roles/openshift_aws/tasks/provision_nodes.yml index 3e84666a2..041ed0791 100644 --- a/roles/openshift_aws/tasks/provision_nodes.yml +++ b/roles/openshift_aws/tasks/provision_nodes.yml @@ -27,17 +27,17 @@ set_fact: openshift_aws_launch_config_bootstrap_token: "{{ bootstrap['content'] | b64decode }}" -- include: vpc_and_subnet_id.yml +- include_tasks: vpc_and_subnet_id.yml - name: include build compute and infra node groups - include: build_node_group.yml + include_tasks: build_node_group.yml vars: l_nodes_to_build: "{{ openshift_aws_node_group_config }}" l_launch_config_security_groups: "{{ openshift_aws_launch_config_security_groups }}" l_aws_ami_map: "{{ openshift_aws_ami_map }}" - name: include build node group for extra nodes - include: build_node_group.yml + include_tasks: build_node_group.yml when: openshift_aws_node_group_config_extra is defined vars: l_nodes_to_build: "{{ openshift_aws_node_group_config_extra | default({}) }}" @@ -47,4 +47,4 @@ # instances aren't scaling fast enough here, we need to wait for them - when: openshift_aws_wait_for_ssh | bool name: wait for our new nodes to come up - include: wait_for_groups.yml + include_tasks: wait_for_groups.yml diff --git a/roles/openshift_aws/tasks/seal_ami.yml b/roles/openshift_aws/tasks/seal_ami.yml index 0cb749dcc..7a3d0fb68 100644 --- a/roles/openshift_aws/tasks/seal_ami.yml +++ b/roles/openshift_aws/tasks/seal_ami.yml @@ -31,7 +31,7 @@ source-ami: "{{ amioutput.image_id }}" - name: copy the ami for encrypted disks - include: ami_copy.yml + include_tasks: ami_copy.yml vars: openshift_aws_ami_copy_name: "{{ openshift_aws_ami_name }}-encrypted" openshift_aws_ami_copy_src_ami: "{{ amioutput.image_id }}" diff --git a/roles/openshift_aws/tasks/security_group.yml b/roles/openshift_aws/tasks/security_group.yml index 5cc7ae537..43834079e 100644 --- a/roles/openshift_aws/tasks/security_group.yml +++ b/roles/openshift_aws/tasks/security_group.yml @@ -6,11 +6,11 @@ "tag:Name": "{{ openshift_aws_clusterid }}" register: vpcout -- include: security_group_create.yml +- include_tasks: security_group_create.yml vars: l_security_groups: "{{ openshift_aws_node_security_groups }}" -- include: security_group_create.yml +- include_tasks: security_group_create.yml when: openshift_aws_node_security_groups_extra is defined vars: l_security_groups: "{{ openshift_aws_node_security_groups_extra | default({}) }}" diff --git a/roles/openshift_aws/tasks/upgrade_node_group.yml b/roles/openshift_aws/tasks/upgrade_node_group.yml index d7851d887..c3f86f523 100644 --- a/roles/openshift_aws/tasks/upgrade_node_group.yml +++ b/roles/openshift_aws/tasks/upgrade_node_group.yml @@ -4,13 +4,13 @@ when: - openshift_aws_current_version == openshift_aws_new_version -- include: provision_nodes.yml +- include_tasks: provision_nodes.yml -- include: accept_nodes.yml +- include_tasks: accept_nodes.yml -- include: setup_scale_group_facts.yml +- include_tasks: setup_scale_group_facts.yml -- include: setup_master_group.yml +- include_tasks: setup_master_group.yml vars: # we do not set etcd here as its limited to 1 or 3 openshift_aws_masters_groups: masters,nodes diff --git a/roles/openshift_cluster_autoscaler/tasks/main.yml b/roles/openshift_cluster_autoscaler/tasks/main.yml index 173dcf044..ca7dfb885 100644 --- a/roles/openshift_cluster_autoscaler/tasks/main.yml +++ b/roles/openshift_cluster_autoscaler/tasks/main.yml @@ -31,7 +31,7 @@ type: role name: "{{ openshift_cluster_autoscaler_name }}" -- include: aws.yml +- include_tasks: aws.yml when: openshift_cluster_autoscaler_cloud_provider == 'aws' - name: create the policies diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py index 6a1a08047..d2bd7357a 100755 --- a/roles/openshift_facts/library/openshift_facts.py +++ b/roles/openshift_facts/library/openshift_facts.py @@ -94,8 +94,7 @@ def migrate_admission_plugin_facts(facts): # Merge existing kube_admission_plugin_config with admission_plugin_config. facts['master']['admission_plugin_config'] = merge_facts(facts['master']['admission_plugin_config'], facts['master']['kube_admission_plugin_config'], - additive_facts_to_overwrite=[], - protected_facts_to_overwrite=[]) + additive_facts_to_overwrite=[]) # Remove kube_admission_plugin_config fact facts['master'].pop('kube_admission_plugin_config', None) return facts @@ -854,7 +853,7 @@ values provided as a list. Hence the gratuitous use of ['foo'] below. # If we've added items to the kubelet_args dict then we need # to merge the new items back into the main facts object. if kubelet_args != {}: - facts = merge_facts({'node': {'kubelet_args': kubelet_args}}, facts, [], []) + facts = merge_facts({'node': {'kubelet_args': kubelet_args}}, facts, []) return facts @@ -876,7 +875,7 @@ def build_controller_args(facts): controller_args['cloud-provider'] = ['gce'] controller_args['cloud-config'] = [cloud_cfg_path + '/gce.conf'] if controller_args != {}: - facts = merge_facts({'master': {'controller_args': controller_args}}, facts, [], []) + facts = merge_facts({'master': {'controller_args': controller_args}}, facts, []) return facts @@ -898,7 +897,7 @@ def build_api_server_args(facts): api_server_args['cloud-provider'] = ['gce'] api_server_args['cloud-config'] = [cloud_cfg_path + '/gce.conf'] if api_server_args != {}: - facts = merge_facts({'master': {'api_server_args': api_server_args}}, facts, [], []) + facts = merge_facts({'master': {'api_server_args': api_server_args}}, facts, []) return facts @@ -1085,7 +1084,7 @@ def apply_provider_facts(facts, provider_facts): # Disabling pylint too many branches. This function needs refactored # but is a very core part of openshift_facts. # pylint: disable=too-many-branches, too-many-nested-blocks -def merge_facts(orig, new, additive_facts_to_overwrite, protected_facts_to_overwrite): +def merge_facts(orig, new, additive_facts_to_overwrite): """ Recursively merge facts dicts Args: @@ -1093,14 +1092,11 @@ def merge_facts(orig, new, additive_facts_to_overwrite, protected_facts_to_overw new (dict): facts to update additive_facts_to_overwrite (list): additive facts to overwrite in jinja '.' notation ex: ['master.named_certificates'] - protected_facts_to_overwrite (list): protected facts to overwrite in jinja - '.' notation ex: ['master.master_count'] Returns: dict: the merged facts """ additive_facts = ['named_certificates'] - protected_facts = ['ha'] # Facts we do not ever want to merge. These originate in inventory variables # and contain JSON dicts. We don't ever want to trigger a merge @@ -1132,14 +1128,7 @@ def merge_facts(orig, new, additive_facts_to_overwrite, protected_facts_to_overw if '.' in item and item.startswith(key + '.'): relevant_additive_facts.append(item) - # Collect the subset of protected facts to overwrite - # if key matches. These will be passed to the - # subsequent merge_facts call. - relevant_protected_facts = [] - for item in protected_facts_to_overwrite: - if '.' in item and item.startswith(key + '.'): - relevant_protected_facts.append(item) - facts[key] = merge_facts(value, new[key], relevant_additive_facts, relevant_protected_facts) + facts[key] = merge_facts(value, new[key], relevant_additive_facts) # Key matches an additive fact and we are not overwriting # it so we will append the new value to the existing value. elif key in additive_facts and key not in [x.split('.')[-1] for x in additive_facts_to_overwrite]: @@ -1149,18 +1138,6 @@ def merge_facts(orig, new, additive_facts_to_overwrite, protected_facts_to_overw if item not in new_fact: new_fact.append(item) facts[key] = new_fact - # Key matches a protected fact and we are not overwriting - # it so we will determine if it is okay to change this - # fact. - elif key in protected_facts and key not in [x.split('.')[-1] for x in protected_facts_to_overwrite]: - # ha (bool) can not change unless it has been passed - # as a protected fact to overwrite. - if key == 'ha': - if safe_get_bool(value) != safe_get_bool(new[key]): - # pylint: disable=line-too-long - module.fail_json(msg='openshift_facts received a different value for openshift.master.ha') # noqa: F405 - else: - facts[key] = value # No other condition has been met. Overwrite the old fact # with the new value. else: @@ -1493,8 +1470,6 @@ class OpenShiftFacts(object): local_facts (dict): local facts to set additive_facts_to_overwrite (list): additive facts to overwrite in jinja '.' notation ex: ['master.named_certificates'] - protected_facts_to_overwrite (list): protected facts to overwrite in jinja - '.' notation ex: ['master.master_count'] Raises: OpenShiftFactsUnsupportedRoleError: @@ -1512,8 +1487,7 @@ class OpenShiftFacts(object): def __init__(self, role, filename, local_facts, additive_facts_to_overwrite=None, openshift_env=None, - openshift_env_structures=None, - protected_facts_to_overwrite=None): + openshift_env_structures=None): self.changed = False self.filename = filename if role not in self.known_roles: @@ -1537,15 +1511,13 @@ class OpenShiftFacts(object): self.facts = self.generate_facts(local_facts, additive_facts_to_overwrite, openshift_env, - openshift_env_structures, - protected_facts_to_overwrite) + openshift_env_structures) def generate_facts(self, local_facts, additive_facts_to_overwrite, openshift_env, - openshift_env_structures, - protected_facts_to_overwrite): + openshift_env_structures): """ Generate facts Args: @@ -1553,16 +1525,13 @@ class OpenShiftFacts(object): additive_facts_to_overwrite (list): additive facts to overwrite in jinja '.' notation ex: ['master.named_certificates'] openshift_env (dict): openshift_env facts for overriding generated defaults - protected_facts_to_overwrite (list): protected facts to overwrite in jinja - '.' notation ex: ['master.master_count'] Returns: dict: The generated facts """ local_facts = self.init_local_facts(local_facts, additive_facts_to_overwrite, openshift_env, - openshift_env_structures, - protected_facts_to_overwrite) + openshift_env_structures) roles = local_facts.keys() if 'common' in local_facts and 'deployment_type' in local_facts['common']: @@ -1580,8 +1549,7 @@ class OpenShiftFacts(object): facts = apply_provider_facts(defaults, provider_facts) facts = merge_facts(facts, local_facts, - additive_facts_to_overwrite, - protected_facts_to_overwrite) + additive_facts_to_overwrite) facts = migrate_oauth_template_facts(facts) facts['current_config'] = get_current_config(facts) facts = set_url_facts_if_unset(facts) @@ -1777,8 +1745,7 @@ class OpenShiftFacts(object): def init_local_facts(self, facts=None, additive_facts_to_overwrite=None, openshift_env=None, - openshift_env_structures=None, - protected_facts_to_overwrite=None): + openshift_env_structures=None): """ Initialize the local facts Args: @@ -1786,8 +1753,6 @@ class OpenShiftFacts(object): additive_facts_to_overwrite (list): additive facts to overwrite in jinja '.' notation ex: ['master.named_certificates'] openshift_env (dict): openshift env facts to set - protected_facts_to_overwrite (list): protected facts to overwrite in jinja - '.' notation ex: ['master.master_count'] Returns: @@ -1829,8 +1794,7 @@ class OpenShiftFacts(object): facts_to_set = merge_facts(orig=facts_to_set, new=oo_env_facts, - additive_facts_to_overwrite=[], - protected_facts_to_overwrite=[]) + additive_facts_to_overwrite=[]) local_facts = get_local_facts_from_file(self.filename) @@ -1838,8 +1802,7 @@ class OpenShiftFacts(object): new_local_facts = merge_facts(migrated_facts, facts_to_set, - additive_facts_to_overwrite, - protected_facts_to_overwrite) + additive_facts_to_overwrite) new_local_facts = self.remove_empty_facts(new_local_facts) @@ -1948,8 +1911,7 @@ def main(): local_facts=dict(default=None, type='dict', required=False), additive_facts_to_overwrite=dict(default=[], type='list', required=False), openshift_env=dict(default={}, type='dict', required=False), - openshift_env_structures=dict(default=[], type='list', required=False), - protected_facts_to_overwrite=dict(default=[], type='list', required=False) + openshift_env_structures=dict(default=[], type='list', required=False) ), supports_check_mode=True, add_file_common_args=True, @@ -1967,7 +1929,6 @@ def main(): additive_facts_to_overwrite = module.params['additive_facts_to_overwrite'] # noqa: F405 openshift_env = module.params['openshift_env'] # noqa: F405 openshift_env_structures = module.params['openshift_env_structures'] # noqa: F405 - protected_facts_to_overwrite = module.params['protected_facts_to_overwrite'] # noqa: F405 fact_file = '/etc/ansible/facts.d/openshift.fact' @@ -1976,8 +1937,7 @@ def main(): local_facts, additive_facts_to_overwrite, openshift_env, - openshift_env_structures, - protected_facts_to_overwrite) + openshift_env_structures) file_params = module.params.copy() # noqa: F405 file_params['path'] = fact_file diff --git a/roles/openshift_hosted/tasks/registry.yml b/roles/openshift_hosted/tasks/registry.yml index 4797fb788..de302c740 100644 --- a/roles/openshift_hosted/tasks/registry.yml +++ b/roles/openshift_hosted/tasks/registry.yml @@ -6,7 +6,7 @@ check_mode: no - name: setup firewall - include: firewall.yml + import_tasks: firewall.yml vars: l_openshift_hosted_firewall_enabled: "{{ r_openshift_hosted_registry_firewall_enabled }}" l_openshift_hosted_use_firewalld: "{{ r_openshift_hosted_registry_use_firewalld }}" @@ -94,14 +94,12 @@ service_type: ClusterIP clusterip: '{{ openshift_hosted_registry_clusterip | default(omit) }}' -- include: secure.yml - static: no +- include_tasks: secure.yml run_once: true when: - not (openshift_docker_hosted_registry_insecure | default(False)) | bool -- include: storage/object_storage.yml - static: no +- include_tasks: storage/object_storage.yml when: - openshift_hosted_registry_storage_kind | default(none) == 'object' @@ -116,7 +114,7 @@ when: - openshift_hosted_registry_storage_kind | default(none) in ['nfs', 'openstack', 'glusterfs'] -- include: storage/glusterfs_endpoints.yml +- include_tasks: storage/glusterfs_endpoints.yml when: - openshift_hosted_registry_storage_glusterfs_ips|length > 0 - openshift_hosted_registry_storage_kind | default(none) in ['glusterfs'] @@ -141,12 +139,12 @@ namespace: "{{ openshift_hosted_registry_namespace }}" - name: Wait for pod (Registry) - include: wait_for_pod.yml + include_tasks: wait_for_pod.yml vars: l_openshift_hosted_wait_for_pod: "{{ openshift_hosted_registry_wait }}" l_openshift_hosted_wfp_items: "{{ r_openshift_hosted_registry_list }}" -- include: storage/glusterfs.yml +- include_tasks: storage/glusterfs.yml when: - openshift_hosted_registry_storage_kind | default(none) == 'glusterfs' or openshift_hosted_registry_storage_glusterfs_swap diff --git a/roles/openshift_hosted/tasks/router.yml b/roles/openshift_hosted/tasks/router.yml index 57c10b637..4e9219477 100644 --- a/roles/openshift_hosted/tasks/router.yml +++ b/roles/openshift_hosted/tasks/router.yml @@ -1,6 +1,6 @@ --- - name: setup firewall - include: firewall.yml + import_tasks: firewall.yml vars: l_openshift_hosted_firewall_enabled: "{{ r_openshift_hosted_router_firewall_enabled }}" l_openshift_hosted_use_firewalld: "{{ r_openshift_hosted_router_use_firewalld }}" @@ -100,7 +100,7 @@ with_items: "{{ openshift_hosted_routers }}" - name: Wait for pod (Routers) - include: wait_for_pod.yml + include_tasks: wait_for_pod.yml vars: l_openshift_hosted_wait_for_pod: "{{ openshift_hosted_router_wait }}" l_openshift_hosted_wfp_items: "{{ openshift_hosted_routers }}" diff --git a/roles/openshift_hosted/tasks/secure.yml b/roles/openshift_hosted/tasks/secure.yml index ecbf5b141..378ae32dc 100644 --- a/roles/openshift_hosted/tasks/secure.yml +++ b/roles/openshift_hosted/tasks/secure.yml @@ -1,12 +1,10 @@ --- - name: Include reencrypt route configuration - include: secure/reencrypt.yml - static: no + include_tasks: secure/reencrypt.yml when: openshift_hosted_registry_routetermination == 'reencrypt' - name: Include passthrough route configuration - include: secure/passthrough.yml - static: no + include_tasks: secure/passthrough.yml when: openshift_hosted_registry_routetermination == 'passthrough' - name: Fetch the docker-registry route diff --git a/roles/openshift_hosted/tasks/storage/object_storage.yml b/roles/openshift_hosted/tasks/storage/object_storage.yml index 3d1b2c68e..a8c26fb51 100644 --- a/roles/openshift_hosted/tasks/storage/object_storage.yml +++ b/roles/openshift_hosted/tasks/storage/object_storage.yml @@ -1,5 +1,5 @@ --- -- include: s3.yml +- include_tasks: s3.yml when: openshift_hosted_registry_storage_provider == 's3' - name: Ensure the registry secret exists diff --git a/roles/openshift_logging/tasks/generate_certs.yaml b/roles/openshift_logging/tasks/generate_certs.yaml index f526fd734..082c0128f 100644 --- a/roles/openshift_logging/tasks/generate_certs.yaml +++ b/roles/openshift_logging/tasks/generate_certs.yaml @@ -36,7 +36,7 @@ - top_dir: '{{generated_certs_dir}}' when: not signing_conf_file.stat.exists -- include: procure_server_certs.yaml +- include_tasks: procure_server_certs.yaml loop_control: loop_var: cert_info with_items: @@ -45,7 +45,7 @@ - procure_component: kibana-internal hostnames: "kibana, kibana-ops, {{openshift_logging_kibana_hostname}}, {{openshift_logging_kibana_ops_hostname}}" -- include: procure_server_certs.yaml +- include_tasks: procure_server_certs.yaml loop_control: loop_var: cert_info with_items: @@ -53,14 +53,14 @@ hostnames: "logging-mux, {{openshift_logging_mux_hostname}}" when: openshift_logging_use_mux | bool -- include: procure_shared_key.yaml +- include_tasks: procure_shared_key.yaml loop_control: loop_var: shared_key_info with_items: - procure_component: mux when: openshift_logging_use_mux | bool -- include: procure_server_certs.yaml +- include_tasks: procure_server_certs.yaml loop_control: loop_var: cert_info with_items: @@ -68,7 +68,7 @@ hostnames: "es, {{openshift_logging_es_hostname}}" when: openshift_logging_es_allow_external | bool -- include: procure_server_certs.yaml +- include_tasks: procure_server_certs.yaml loop_control: loop_var: cert_info with_items: @@ -109,7 +109,7 @@ - not ca_cert_srl_file.stat.exists - name: Generate PEM certs - include: generate_pems.yaml component={{node_name}} + include_tasks: generate_pems.yaml component={{node_name}} with_items: - system.logging.fluentd - system.logging.kibana @@ -119,7 +119,7 @@ loop_var: node_name - name: Generate PEM cert for mux - include: generate_pems.yaml component={{node_name}} + include_tasks: generate_pems.yaml component={{node_name}} with_items: - system.logging.mux loop_control: @@ -127,7 +127,7 @@ when: openshift_logging_use_mux | bool - name: Generate PEM cert for Elasticsearch external route - include: generate_pems.yaml component={{node_name}} + include_tasks: generate_pems.yaml component={{node_name}} with_items: - system.logging.es loop_control: @@ -135,7 +135,7 @@ when: openshift_logging_es_allow_external | bool - name: Creating necessary JKS certs - include: generate_jks.yaml + include_tasks: generate_jks.yaml # TODO: make idempotent - name: Generate proxy session diff --git a/roles/openshift_logging/tasks/install_logging.yaml b/roles/openshift_logging/tasks/install_logging.yaml index 2fefdc894..bb8ebec6b 100644 --- a/roles/openshift_logging/tasks/install_logging.yaml +++ b/roles/openshift_logging/tasks/install_logging.yaml @@ -52,7 +52,7 @@ changed_when: False check_mode: no -- include: generate_certs.yaml +- include_tasks: generate_certs.yaml vars: generated_certs_dir: "{{openshift.common.config_base}}/logging" @@ -250,7 +250,7 @@ when: - openshift_logging_use_ops | bool -- include: annotate_ops_projects.yaml +- include_tasks: annotate_ops_projects.yaml ## Curator - include_role: @@ -311,4 +311,4 @@ openshift_logging_install_eventrouter | default(false) | bool -- include: update_master_config.yaml +- include_tasks: update_master_config.yaml diff --git a/roles/openshift_logging/tasks/main.yaml b/roles/openshift_logging/tasks/main.yaml index 7f8e88036..91db457d1 100644 --- a/roles/openshift_logging/tasks/main.yaml +++ b/roles/openshift_logging/tasks/main.yaml @@ -19,11 +19,11 @@ check_mode: no become: no -- include: install_logging.yaml +- include_tasks: install_logging.yaml when: - openshift_logging_install_logging | default(false) | bool -- include: delete_logging.yaml +- include_tasks: delete_logging.yaml when: - not openshift_logging_install_logging | default(false) | bool diff --git a/roles/openshift_logging_curator/tasks/main.yaml b/roles/openshift_logging_curator/tasks/main.yaml index 7ddf57450..e7ef5ff22 100644 --- a/roles/openshift_logging_curator/tasks/main.yaml +++ b/roles/openshift_logging_curator/tasks/main.yaml @@ -12,7 +12,7 @@ openshift_logging_curator_image_prefix: "{{ openshift_logging_curator_image_prefix | default(__openshift_logging_curator_image_prefix) }}" openshift_logging_curator_image_version: "{{ openshift_logging_curator_image_version | default(__openshift_logging_curator_image_version) }}" -- include: determine_version.yaml +- include_tasks: determine_version.yaml # allow passing in a tempdir - name: Create temp directory for doing work in diff --git a/roles/openshift_logging_elasticsearch/tasks/main.yaml b/roles/openshift_logging_elasticsearch/tasks/main.yaml index 770892d52..8f2050043 100644 --- a/roles/openshift_logging_elasticsearch/tasks/main.yaml +++ b/roles/openshift_logging_elasticsearch/tasks/main.yaml @@ -30,7 +30,7 @@ openshift_logging_elasticsearch_image_prefix: "{{ openshift_logging_elasticsearch_image_prefix | default(__openshift_logging_elasticsearch_image_prefix) }}" openshift_logging_elasticsearch_image_version: "{{ openshift_logging_elasticsearch_image_version | default(__openshift_logging_elasticsearch_image_version) }}" -- include: determine_version.yaml +- include_tasks: determine_version.yaml # allow passing in a tempdir - name: Create temp directory for doing work in diff --git a/roles/openshift_logging_eventrouter/tasks/main.yaml b/roles/openshift_logging_eventrouter/tasks/main.yaml index b1f93eeb9..96b181d61 100644 --- a/roles/openshift_logging_eventrouter/tasks/main.yaml +++ b/roles/openshift_logging_eventrouter/tasks/main.yaml @@ -12,8 +12,8 @@ openshift_logging_eventrouter_image_prefix: "{{ openshift_logging_eventrouter_image_prefix | default(__openshift_logging_eventrouter_image_prefix) }}" openshift_logging_eventrouter_image_version: "{{ openshift_logging_eventrouter_image_version | default(__openshift_logging_eventrouter_image_version) }}" -- include: "{{ role_path }}/tasks/install_eventrouter.yaml" +- include_tasks: install_eventrouter.yaml when: openshift_logging_install_eventrouter | default(false) | bool -- include: "{{ role_path }}/tasks/delete_eventrouter.yaml" +- include_tasks: delete_eventrouter.yaml when: not openshift_logging_install_eventrouter | default(false) | bool diff --git a/roles/openshift_logging_fluentd/tasks/main.yaml b/roles/openshift_logging_fluentd/tasks/main.yaml index f8683ab75..87eedfb4b 100644 --- a/roles/openshift_logging_fluentd/tasks/main.yaml +++ b/roles/openshift_logging_fluentd/tasks/main.yaml @@ -47,7 +47,7 @@ openshift_logging_fluentd_image_prefix: "{{ openshift_logging_fluentd_image_prefix | default(__openshift_logging_fluentd_image_prefix) }}" openshift_logging_fluentd_image_version: "{{ openshift_logging_fluentd_image_version | default(__openshift_logging_fluentd_image_version) }}" -- include: determine_version.yaml +- include_tasks: determine_version.yaml # allow passing in a tempdir - name: Create temp directory for doing work in @@ -216,7 +216,7 @@ openshift_logging_fluentd_hosts: "{{ fluentd_hosts.results.results[0]['items'] | map(attribute='metadata.name') | list }}" when: "'--all' in openshift_logging_fluentd_hosts" -- include: label_and_wait.yaml +- include_tasks: label_and_wait.yaml vars: node: "{{ fluentd_host }}" with_items: "{{ openshift_logging_fluentd_hosts }}" diff --git a/roles/openshift_logging_kibana/tasks/main.yaml b/roles/openshift_logging_kibana/tasks/main.yaml index 9d99114c5..77bf8042a 100644 --- a/roles/openshift_logging_kibana/tasks/main.yaml +++ b/roles/openshift_logging_kibana/tasks/main.yaml @@ -15,7 +15,7 @@ openshift_logging_kibana_proxy_image_prefix: "{{ openshift_logging_kibana_proxy_image_prefix | default(__openshift_logging_kibana_proxy_image_prefix) }}" openshift_logging_kibana_proxy_image_version: "{{ openshift_logging_kibana_proxy_image_version | default(__openshift_logging_kibana_proxy_image_version) }}" -- include: determine_version.yaml +- include_tasks: determine_version.yaml # allow passing in a tempdir - name: Create temp directory for doing work in diff --git a/roles/openshift_logging_mux/tasks/main.yaml b/roles/openshift_logging_mux/tasks/main.yaml index 242d92188..68948bce2 100644 --- a/roles/openshift_logging_mux/tasks/main.yaml +++ b/roles/openshift_logging_mux/tasks/main.yaml @@ -20,7 +20,7 @@ openshift_logging_mux_image_prefix: "{{ openshift_logging_mux_image_prefix | default(__openshift_logging_mux_image_prefix) }}" openshift_logging_mux_image_version: "{{ openshift_logging_mux_image_version | default(__openshift_logging_mux_image_version) }}" -- include: determine_version.yaml +- include_tasks: determine_version.yaml # allow passing in a tempdir - name: Create temp directory for doing work in diff --git a/roles/openshift_management/tasks/main.yml b/roles/openshift_management/tasks/main.yml index 3bade9e8c..f212dba7c 100644 --- a/roles/openshift_management/tasks/main.yml +++ b/roles/openshift_management/tasks/main.yml @@ -3,7 +3,7 @@ # Users, projects, and privileges - name: Run pre-install Management validation checks - include: validate.yml + include_tasks: validate.yml # This creates a service account allowing Container Provider # integration (managing OCP/Origin via MIQ/Management) @@ -18,18 +18,18 @@ display_name: "{{ openshift_management_project_description }}" - name: Create and Authorize Management Accounts - include: accounts.yml + include_tasks: accounts.yml ###################################################################### # STORAGE - Initialize basic storage class - name: Determine the correct NFS host if required - include: storage/nfs_server.yml + include_tasks: storage/nfs_server.yml when: openshift_management_storage_class in ['nfs', 'nfs_external'] #--------------------------------------------------------------------- # * nfs - set up NFS shares on the first master for a proof of concept - name: Create required NFS exports for Management app storage - include: storage/nfs.yml + include_tasks: storage/nfs.yml when: openshift_management_storage_class == 'nfs' #--------------------------------------------------------------------- @@ -56,14 +56,14 @@ ###################################################################### # APPLICATION TEMPLATE - name: Install the Management app and PV templates - include: template.yml + include_tasks: template.yml ###################################################################### # APP & DB Storage # For local/external NFS backed installations - name: "Create the required App and DB PVs using {{ openshift_management_storage_class }}" - include: storage/create_nfs_pvs.yml + include_tasks: storage/create_nfs_pvs.yml when: - openshift_management_storage_class in ['nfs', 'nfs_external'] diff --git a/roles/openshift_management/tasks/storage/storage.yml b/roles/openshift_management/tasks/storage/storage.yml index d8bf7aa3e..a3675b29b 100644 --- a/roles/openshift_management/tasks/storage/storage.yml +++ b/roles/openshift_management/tasks/storage/storage.yml @@ -1,3 +1,3 @@ --- -- include: nfs.yml +- include_tasks: nfs.yml when: not (openshift_cloudprovider_kind is defined and (openshift_cloudprovider_kind == 'aws' or openshift_cloudprovider_kind == 'gce')) diff --git a/roles/openshift_master/tasks/system_container.yml b/roles/openshift_master/tasks/system_container.yml index 23386f11b..450f6d803 100644 --- a/roles/openshift_master/tasks/system_container.yml +++ b/roles/openshift_master/tasks/system_container.yml @@ -1,8 +1,4 @@ --- -- name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - name: Pre-pull master system container image command: > diff --git a/roles/openshift_nfs/tasks/setup.yml b/roles/openshift_nfs/tasks/setup.yml index edb854467..1aa7e7079 100644 --- a/roles/openshift_nfs/tasks/setup.yml +++ b/roles/openshift_nfs/tasks/setup.yml @@ -1,7 +1,6 @@ --- - name: setup firewall - include: firewall.yml - static: yes + import_tasks: firewall.yml - name: Install nfs-utils package: name=nfs-utils state=present diff --git a/roles/openshift_node/tasks/main.yml b/roles/openshift_node/tasks/main.yml index e60d96760..d9f3e920d 100644 --- a/roles/openshift_node/tasks/main.yml +++ b/roles/openshift_node/tasks/main.yml @@ -6,7 +6,7 @@ - deployment_type == 'openshift-enterprise' - not openshift_use_crio -- include: dnsmasq.yml +- include_tasks: dnsmasq.yml - name: setup firewall import_tasks: firewall.yml @@ -44,13 +44,6 @@ - name: include node installer include_tasks: install.yml -- name: Restart cri-o - systemd: - name: cri-o - enabled: yes - state: restarted - when: openshift_use_crio - - name: restart NetworkManager to ensure resolv.conf is present systemd: name: NetworkManager diff --git a/roles/openshift_node/tasks/node_system_container.yml b/roles/openshift_node/tasks/node_system_container.yml index eb8d9a6a5..98a391890 100644 --- a/roles/openshift_node/tasks/node_system_container.yml +++ b/roles/openshift_node/tasks/node_system_container.yml @@ -1,8 +1,4 @@ --- -- name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - name: Pre-pull node system container image command: > diff --git a/roles/openshift_node/tasks/openvswitch_system_container.yml b/roles/openshift_node/tasks/openvswitch_system_container.yml index d33e172c1..b61bc84c1 100644 --- a/roles/openshift_node/tasks/openvswitch_system_container.yml +++ b/roles/openshift_node/tasks/openvswitch_system_container.yml @@ -7,11 +7,6 @@ l_service_name: "{{ openshift_docker_service_name }}" when: not openshift_use_crio -- name: Ensure proxies are in the atomic.conf - include_role: - name: openshift_atomic - tasks_from: proxy - - name: Pre-pull OpenVSwitch system container image command: > atomic pull --storage=ostree {{ 'docker:' if system_images_registry == 'docker' else system_images_registry + '/' }}{{ openshift.node.ovs_system_image }}:{{ openshift_image_tag }} diff --git a/roles/openshift_openstack/tasks/check-prerequisites.yml b/roles/openshift_openstack/tasks/check-prerequisites.yml index 0ef60459f..30996cc47 100644 --- a/roles/openshift_openstack/tasks/check-prerequisites.yml +++ b/roles/openshift_openstack/tasks/check-prerequisites.yml @@ -87,7 +87,7 @@ msg: "Keypair {{ openshift_openstack_keypair_name }} is not available" # Check that custom images are available -- include: custom_image_check.yaml +- include_tasks: custom_image_check.yaml with_items: - "{{ openshift_openstack_master_image }}" - "{{ openshift_openstack_infra_image }}" @@ -96,7 +96,7 @@ - "{{ openshift_openstack_etcd_image }}" # Check that custom flavors are available -- include: custom_flavor_check.yaml +- include_tasks: custom_flavor_check.yaml with_items: - "{{ openshift_openstack_master_flavor }}" - "{{ openshift_openstack_infra_flavor }}" diff --git a/roles/openshift_openstack/tasks/node-configuration.yml b/roles/openshift_openstack/tasks/node-configuration.yml index 53e3bb28c..59df2e396 100644 --- a/roles/openshift_openstack/tasks/node-configuration.yml +++ b/roles/openshift_openstack/tasks/node-configuration.yml @@ -4,6 +4,6 @@ msg: "SELinux is required for OpenShift and has been detected as '{{ ansible_selinux.config_mode }}'" when: ansible_selinux.config_mode != "enforcing" -- include: container-storage-setup.yml +- include_tasks: container-storage-setup.yml -- include: node-network.yml +- include_tasks: node-network.yml diff --git a/roles/openshift_openstack/tasks/provision.yml b/roles/openshift_openstack/tasks/provision.yml index dccbe334c..b774bd620 100644 --- a/roles/openshift_openstack/tasks/provision.yml +++ b/roles/openshift_openstack/tasks/provision.yml @@ -1,6 +1,6 @@ --- - name: Generate the templates - include: generate-templates.yml + include_tasks: generate-templates.yml when: - openshift_openstack_stack_state == 'present' @@ -17,7 +17,7 @@ meta: refresh_inventory - name: CleanUp - include: cleanup.yml + include_tasks: cleanup.yml when: - openshift_openstack_stack_state == 'present' diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml index 4b33e91b4..315bc5614 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -82,7 +82,7 @@ retries: "{{ (glusterfs_timeout | int / 10) | int }}" when: glusterfs_heketi_wipe -- include: glusterfs_deploy.yml +- include_tasks: glusterfs_deploy.yml when: glusterfs_is_native - name: Create heketi service account @@ -212,7 +212,7 @@ when: - glusterfs_heketi_is_native -- include: heketi_deploy_part1.yml +- include_tasks: heketi_deploy_part1.yml when: - glusterfs_heketi_is_native - glusterfs_heketi_deploy_is_missing @@ -256,7 +256,7 @@ when: - glusterfs_heketi_topology_load -- include: heketi_deploy_part2.yml +- include_tasks: heketi_deploy_part2.yml when: - glusterfs_heketi_is_native - glusterfs_heketi_is_missing @@ -312,8 +312,8 @@ when: - glusterfs_storageclass or glusterfs_s3_deploy -- include: glusterblock_deploy.yml +- include_tasks: glusterblock_deploy.yml when: glusterfs_block_deploy -- include: gluster_s3_deploy.yml +- include_tasks: gluster_s3_deploy.yml when: glusterfs_s3_deploy diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml index 71c1311cd..73b9791eb 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -46,4 +46,4 @@ glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_heketi_fstab }}" glusterfs_nodes: "{{ groups.glusterfs | default([]) }}" -- include: glusterfs_common.yml +- include_tasks: glusterfs_common.yml diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml index fa50e39a2..7466702b8 100644 --- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -46,7 +46,7 @@ glusterfs_heketi_fstab: "{{ openshift_storage_glusterfs_registry_heketi_fstab }}" glusterfs_nodes: "{% if groups.glusterfs_registry is defined %}{% set nodes = groups.glusterfs_registry %}{% elif 'groups.glusterfs' is defined %}{% set nodes = groups.glusterfs %}{% else %}{% set nodes = '[]' %}{% endif %}{{ nodes }}" -- include: glusterfs_common.yml +- include_tasks: glusterfs_common.yml when: - glusterfs_nodes | default([]) | count > 0 - "'glusterfs' not in groups or glusterfs_nodes != groups.glusterfs" diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml index 728f15a21..b48bfc88e 100644 --- a/roles/openshift_storage_glusterfs/tasks/main.yml +++ b/roles/openshift_storage_glusterfs/tasks/main.yml @@ -5,11 +5,11 @@ changed_when: False check_mode: no -- include: glusterfs_config.yml +- include_tasks: glusterfs_config.yml when: - groups.glusterfs | default([]) | count > 0 -- include: glusterfs_registry.yml +- include_tasks: glusterfs_registry.yml when: > groups.glusterfs_registry | default([]) | count > 0 or (openshift_hosted_registry_storage_kind | default(none) == 'glusterfs') diff --git a/roles/openshift_storage_nfs_lvm/tasks/main.yml b/roles/openshift_storage_nfs_lvm/tasks/main.yml index 49dd657b5..c8e7b6d7c 100644 --- a/roles/openshift_storage_nfs_lvm/tasks/main.yml +++ b/roles/openshift_storage_nfs_lvm/tasks/main.yml @@ -20,7 +20,7 @@ file: path={{osnl_mount_dir}}/{{ item }} owner=nfsnobody group=nfsnobody mode=0700 with_sequence: start={{osnl_volume_num_start}} count={{osnl_number_of_volumes}} format={{osnl_volume_prefix}}{{osnl_volume_size}}g%04d -- include: nfs.yml +- include_tasks: nfs.yml - name: Create volume json file template: src=../templates/nfs.json.j2 dest=/root/persistent-volume.{{ item }}.json diff --git a/roles/rhel_subscribe/tasks/main.yml b/roles/rhel_subscribe/tasks/main.yml index 9ca49b569..f83cf9157 100644 --- a/roles/rhel_subscribe/tasks/main.yml +++ b/roles/rhel_subscribe/tasks/main.yml @@ -69,7 +69,7 @@ until: subscribe_pool | succeeded when: openshift_pool_id.stdout != '' -- include: enterprise.yml +- include_tasks: enterprise.yml when: - deployment_type == 'openshift-enterprise' - not ostree_booted.stat.exists | bool |