Cleanup ovs file and restart docker on every upgrade.

In 3.3 one of our services lays down a systemd drop-in for configuring
Docker networking to use lbr0. In 3.4, this has been changed but the
file must be cleaned up manually by us.

However, after removing the file docker requires a restart. This had big
implications particularly in containerized environments where upgrade is
a very fragile series of upgrading and service restarts.

To avoid double docker restarts, and thus double service restarts in
containerized environments, this change does the following:

- Skip restart during docker upgrade, if it is required. We will restart
  on our own later.
- Skip containerized service restarts when we upgrade the services
  themselves.
- Clean shutdown of all containerized services.
- Restart Docker. (always, previously this only happened if it needed an
  upgrade)
- Ensure all containerized services are restarted.
- Restart rpm node services. (always)
- Mark node schedulable again.

At the end of this process, docker0 should be back on the system.

2 files changed, 31 insertions, 27 deletions

diff --git a/playbooks/common/openshift-cluster/upgrades/docker/restart.yml b/playbooks/common/openshift-cluster/upgrades/docker/restart.yml
new file mode 100644
index 000000000..d800b289b
--- /dev/null
+++ b/playbooks/common/openshift-cluster/upgrades/docker/restart.yml
@@ -0,0 +1,29 @@
+---
+- name: Restart docker
+  service: name=docker state=restarted
+
+- name: Update docker facts
+  openshift_facts:
+    role: docker
+
+- name: Restart containerized services
+  service: name={{ item }} state=started
+  with_items:
+    - etcd_container
+    - openvswitch
+    - "{{ openshift.common.service_type }}-master"
+    - "{{ openshift.common.service_type }}-master-api"
+    - "{{ openshift.common.service_type }}-master-controllers"
+    - "{{ openshift.common.service_type }}-node"
+  failed_when: false
+  when: openshift.common.is_containerized | bool
+
+- name: Wait for master API to come back online
+  become: no
+  local_action:
+    module: wait_for
+      host="{{ inventory_hostname }}"
+      state=started
+      delay=10
+      port="{{ openshift.master.api_port }}"
+  when: inventory_hostname in groups.oo_masters_to_config
diff --git a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
index 5d753447c..44ddf97ad 100644
--- a/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
+++ b/playbooks/common/openshift-cluster/upgrades/docker/upgrade.yml
@@ -37,30 +37,5 @@
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
 
-- service: name=docker state=started
-
-- name: Update docker facts
-  openshift_facts:
-    role: docker
-
-- name: Restart containerized services
-  service: name={{ item }} state=started
-  with_items:
-    - etcd_container
-    - openvswitch
-    - "{{ openshift.common.service_type }}-master"
-    - "{{ openshift.common.service_type }}-master-api"
-    - "{{ openshift.common.service_type }}-master-controllers"
-    - "{{ openshift.common.service_type }}-node"
-  failed_when: false
-  when: openshift.common.is_containerized | bool
-
-- name: Wait for master API to come back online
-  become: no
-  local_action:
-    module: wait_for
-      host="{{ inventory_hostname }}"
-      state=started
-      delay=10
-      port="{{ openshift.master.api_port }}"
-  when: inventory_hostname in groups.oo_masters_to_config
+- include: restart.yml
+  when: not skip_docker_restart | default(False) | bool