Move current node upgrade tasks under openshift_node_upgrade role

6 files changed, 306 insertions, 0 deletions

diff --git a/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml
new file mode 100644
index 000000000..07b0ac715
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/containerized_node_upgrade.yml
@@ -0,0 +1,14 @@
+---
+# This is a hack to allow us to use systemd_units.yml, but skip the handlers which
+# restart services. We will unconditionally restart all containerized services
+# because we have to unconditionally restart Docker:
+- set_fact:
+    skip_node_svc_handlers: True
+
+- name: Update systemd units
+  include: systemd_units.yml
+
+# This is a no-op because of skip_node_svc_handlers, but lets us trigger it before end of
+# play when the node has already been marked schedulable again. (this would look strange
+# in logs otherwise)
+- meta: flush_handlers
diff --git a/roles/openshift_node_upgrade/tasks/docker/restart.yml b/roles/openshift_node_upgrade/tasks/docker/restart.yml
new file mode 100644
index 000000000..176fc3c0b
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/docker/restart.yml
@@ -0,0 +1,33 @@
+---
+# input variables:
+# - openshift.common.service_type
+# - openshift.common.is_containerized
+# - openshift.common.hostname
+# - openshift.master.api_port
+
+- name: Restart docker
+  service: name=docker state=restarted
+
+- name: Update docker facts
+  openshift_facts:
+    role: docker
+
+- name: Restart containerized services
+  service: name={{ item }} state=started
+  with_items:
+    - etcd_container
+    - openvswitch
+    - "{{ openshift.common.service_type }}-master"
+    - "{{ openshift.common.service_type }}-master-api"
+    - "{{ openshift.common.service_type }}-master-controllers"
+    - "{{ openshift.common.service_type }}-node"
+  failed_when: false
+  when: openshift.common.is_containerized | bool
+
+- name: Wait for master API to come back online
+  wait_for:
+    host: "{{ openshift.common.hostname }}"
+    state: started
+    delay: 10
+    port: "{{ openshift.master.api_port }}"
+  when: inventory_hostname in groups.oo_masters_to_config
diff --git a/roles/openshift_node_upgrade/tasks/docker/upgrade.yml b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml
new file mode 100644
index 000000000..089c2aa87
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/docker/upgrade.yml
@@ -0,0 +1,49 @@
+---
+# input variables:
+# - openshift.common.service_type
+# - openshift.common.is_containerized
+# - docker_upgrade_nuke_images
+# - docker_version
+# - skip_docker_restart
+
+# We need docker service up to remove all the images, but these services will keep
+# trying to re-start and thus re-pull the images we're trying to delete.
+- name: Stop containerized services
+  service: name={{ item }} state=stopped
+  with_items:
+    - "{{ openshift.common.service_type }}-master"
+    - "{{ openshift.common.service_type }}-master-api"
+    - "{{ openshift.common.service_type }}-master-controllers"
+    - "{{ openshift.common.service_type }}-node"
+    - etcd_container
+    - openvswitch
+  failed_when: false
+  when: openshift.common.is_containerized | bool
+
+- name: Check Docker image count
+  shell: "docker images -aq | wc -l"
+  register: docker_image_count
+
+- debug: var=docker_image_count.stdout
+
+  # TODO(jchaloup): put all docker_upgrade_nuke_images into a block with only one condition
+- name: Remove all containers and images
+  script: nuke_images.sh
+  register: nuke_images_result
+  when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+- name: Check Docker image count
+  shell: "docker images -aq | wc -l"
+  register: docker_image_count
+  when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+- debug: var=docker_image_count.stdout
+  when: docker_upgrade_nuke_images is defined and docker_upgrade_nuke_images | bool
+
+- service: name=docker state=stopped
+
+- name: Upgrade Docker
+  package: name=docker{{ '-' + docker_version }} state=present
+
+- include: restart.yml
+  when: not skip_docker_restart | default(False) | bool
diff --git a/roles/openshift_node_upgrade/tasks/main.yml b/roles/openshift_node_upgrade/tasks/main.yml
new file mode 100644
index 000000000..b1d5f0e0f
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/main.yml
@@ -0,0 +1,77 @@
+---
+# input variables:
+# - l_docker_upgrade
+# - openshift.common.is_atomic
+# - node_config_hook
+# - openshift_pkg_version
+# - openshift.common.is_containerized
+# - deployment_type
+# - openshift_release
+
+# tasks file for openshift_node_upgrade
+- include: docker/upgrade.yml
+  vars:
+    # We will restart Docker ourselves after everything is ready:
+    skip_docker_restart: True
+  when: l_docker_upgrade is defined and l_docker_upgrade | bool and not openshift.common.is_atomic | bool
+
+- include: "{{ node_config_hook }}"
+  when: node_config_hook is defined
+
+- include: rpm_upgrade.yml
+  vars:
+    component: "node"
+    openshift_version: "{{ openshift_pkg_version | default('') }}"
+  when: not openshift.common.is_containerized | bool
+
+- name: Remove obsolete docker-sdn-ovs.conf
+  file: path=/etc/systemd/system/docker.service.d/docker-sdn-ovs.conf state=absent
+  when: (deployment_type == 'openshift-enterprise' and openshift_release | version_compare('3.4', '>=')) or (deployment_type == 'origin' and openshift_release | version_compare('1.4', '>='))
+
+- include: containerized_node_upgrade.yml
+  when: openshift.common.is_containerized | bool
+
+- name: Ensure containerized services stopped before Docker restart
+  service: name={{ item }} state=stopped
+  with_items:
+  - etcd_container
+  - openvswitch
+  - "{{ openshift.common.service_type }}-master"
+  - "{{ openshift.common.service_type }}-master-api"
+  - "{{ openshift.common.service_type }}-master-controllers"
+  - "{{ openshift.common.service_type }}-node"
+  failed_when: false
+  when: openshift.common.is_containerized | bool
+
+- name: Upgrade openvswitch
+  package:
+    name: openvswitch
+    state: latest
+  register: ovs_pkg
+  when: not openshift.common.is_containerized | bool
+
+- name: Restart openvswitch
+  systemd:
+    name: openvswitch
+    state: restarted
+  when:
+  - not openshift.common.is_containerized | bool
+  - ovs_pkg | changed
+
+# Mandatory Docker restart, ensure all containerized services are running:
+- include: docker/restart.yml
+
+- name: Restart rpm node service
+  service: name="{{ openshift.common.service_type }}-node" state=restarted
+  when: not openshift.common.is_containerized | bool
+
+- name: Wait for node to be ready
+  command: >
+    {{ hostvars[groups.oo_first_master.0].openshift.common.client_binary }} get node {{ openshift.common.hostname | lower }} --no-headers
+  register: node_output
+  delegate_to: "{{ groups.oo_first_master.0 }}"
+  until: "{{ node_output.stdout.split()[1].startswith('Ready')}}"
+  # Give the node two minutes to come back online. Note that we pre-pull images now
+  # so containerized services should restart quickly as well.
+  retries: 24
+  delay: 5
diff --git a/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml
new file mode 100644
index 000000000..480e87d58
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/rpm_upgrade.yml
@@ -0,0 +1,14 @@
+---
+# input variables:
+# - openshift.common.service_type
+# - component
+# - openshift_pkg_version
+# - openshift.common.is_atomic
+
+# We verified latest rpm available is suitable, so just yum update.
+- name: Upgrade packages
+  package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present"
+
+- name: Ensure python-yaml present for config upgrade
+  package: name=PyYAML state=present
+  when: not openshift.common.is_atomic | bool
diff --git a/roles/openshift_node_upgrade/tasks/systemd_units.yml b/roles/openshift_node_upgrade/tasks/systemd_units.yml
new file mode 100644
index 000000000..862cd19c4
--- /dev/null
+++ b/roles/openshift_node_upgrade/tasks/systemd_units.yml
@@ -0,0 +1,119 @@
+---
+# input variables
+# - openshift.node.node_image
+# - openshift_image_tag
+# - openshift.common.is_containerized
+# - openshift.node.ovs_image
+# - openshift.common.use_openshift_sdn
+# - openshift.common.service_type
+# - openshift.node.debug_level
+# - openshift.common.config_base
+# - openshift.common.http_proxy
+# - openshift.common.portal_net
+# - openshift.common
+# - openshift.common.http_proxy
+# notify:
+# - restart openvswitch
+# - restart node
+
+# This file is included both in the openshift_master role and in the upgrade
+# playbooks.
+
+- name: Pre-pull node image
+  command: >
+    docker pull {{ openshift.node.node_image }}:{{ openshift_image_tag }}
+  register: pull_result
+  changed_when: "'Downloaded newer image' in pull_result.stdout"
+  when: openshift.common.is_containerized | bool
+
+- name: Pre-pull openvswitch image
+  command: >
+    docker pull {{ openshift.node.ovs_image }}:{{ openshift_image_tag }}
+  register: pull_result
+  changed_when: "'Downloaded newer image' in pull_result.stdout"
+  when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | bool
+
+- name: Install Node dependencies docker service file
+  template:
+    dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node-dep.service"
+    src: openshift.docker.node.dep.service
+  register: install_node_dep_result
+  when: openshift.common.is_containerized | bool
+
+- name: Install Node docker service file
+  template:
+    dest: "/etc/systemd/system/{{ openshift.common.service_type }}-node.service"
+    src: openshift.docker.node.service
+  register: install_node_result
+  when: openshift.common.is_containerized | bool
+
+- name: Create the openvswitch service env file
+  template:
+    src: openvswitch.sysconfig.j2
+    dest: /etc/sysconfig/openvswitch
+  when: openshift.common.is_containerized | bool
+  register: install_ovs_sysconfig
+  notify:
+  - restart openvswitch
+
+# May be a temporary workaround.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1331590
+- name: Create OpenvSwitch service.d directory
+  file: path=/etc/systemd/system/openvswitch.service.d/ state=directory
+  when: openshift.common.use_openshift_sdn | default(true) | bool
+
+- name: Install OpenvSwitch service OOM fix
+  template:
+    dest: "/etc/systemd/system/openvswitch.service.d/01-avoid-oom.conf"
+    src: openvswitch-avoid-oom.conf
+  when: openshift.common.use_openshift_sdn | default(true) | bool
+  register: install_oom_fix_result
+  notify:
+  - restart openvswitch
+
+- name: Install OpenvSwitch docker service file
+  template:
+    dest: "/etc/systemd/system/openvswitch.service"
+    src: openvswitch.docker.service
+  when: openshift.common.is_containerized | bool and openshift.common.use_openshift_sdn | default(true) | bool
+  notify:
+  - restart openvswitch
+
+- name: Configure Node settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+    create: true
+  with_items:
+  - regex: '^OPTIONS='
+    line: "OPTIONS=--loglevel={{ openshift.node.debug_level | default(2) }}"
+  - regex: '^CONFIG_FILE='
+    line: "CONFIG_FILE={{ openshift.common.config_base }}/node/node-config.yaml"
+  - regex: '^IMAGE_VERSION='
+    line: "IMAGE_VERSION={{ openshift_image_tag }}"
+  notify:
+  - restart node
+
+- name: Configure Proxy Settings
+  lineinfile:
+    dest: /etc/sysconfig/{{ openshift.common.service_type }}-node
+    regexp: "{{ item.regex }}"
+    line: "{{ item.line }}"
+    create: true
+  with_items:
+  - regex: '^HTTP_PROXY='
+    line: "HTTP_PROXY={{ openshift.common.http_proxy | default('') }}"
+  - regex: '^HTTPS_PROXY='
+    line: "HTTPS_PROXY={{ openshift.common.https_proxy | default('') }}"
+  - regex: '^NO_PROXY='
+    line: "NO_PROXY={{ openshift.common.no_proxy | default([]) }},{{ openshift.common.portal_net }},{{ hostvars[groups.oo_first_master.0].openshift.master.sdn_cluster_network_cidr }}"
+  when: ('http_proxy' in openshift.common and openshift.common.http_proxy != '')
+  notify:
+  - restart node
+
+- name: Reload systemd units
+  command: systemctl daemon-reload
+  when: (openshift.common.is_containerized | bool and (install_node_result | changed or install_ovs_sysconfig | changed or install_node_dep_result | changed)) or install_oom_fix_result | changed
+  notify:
+  - restart node