retry package operations

When a package install/update fails due to network blips or other spotty
availability, retry it. If the failure is a real failure (e.g. package
is really not there) it still fails after 3 tries (Ansible default).

11 files changed, 28 insertions, 0 deletions

diff --git a/roles/openshift_node/tasks/bootstrap.yml b/roles/openshift_node/tasks/bootstrap.yml
index ac43ef039..a042bc01b 100644
--- a/roles/openshift_node/tasks/bootstrap.yml
+++ b/roles/openshift_node/tasks/bootstrap.yml
@@ -4,6 +4,8 @@
     name: "{{ item }}"
     state: present
   with_items: "{{ r_openshift_node_image_prep_packages }}"
+  register: result
+  until: result | success
 
 - name: create the directory for node
   file:
diff --git a/roles/openshift_node/tasks/dnsmasq.yml b/roles/openshift_node/tasks/dnsmasq.yml
index 22bdce6c6..f210a3a21 100644
--- a/roles/openshift_node/tasks/dnsmasq.yml
+++ b/roles/openshift_node/tasks/dnsmasq.yml
@@ -13,6 +13,8 @@
 - name: Install dnsmasq
   package: name=dnsmasq state=installed
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: ensure origin/node directory exists
   file:
diff --git a/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
index dede2fb8f..541c8115a 100644
--- a/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
+++ b/roles/openshift_node/tasks/dnsmasq/no-network-manager.yml
@@ -7,5 +7,7 @@
     name: NetworkManager
     state: present
   notify: restart NetworkManager
+  register: result
+  until: result | success
 
 - include_tasks: network-manager.yml
diff --git a/roles/openshift_node/tasks/docker/upgrade.yml b/roles/openshift_node/tasks/docker/upgrade.yml
index ebe87d6fd..d743d2188 100644
--- a/roles/openshift_node/tasks/docker/upgrade.yml
+++ b/roles/openshift_node/tasks/docker/upgrade.yml
@@ -36,5 +36,7 @@
 
 - name: Upgrade Docker
   package: name=docker{{ '-' + docker_version }} state=present
+  register: result
+  until: result | success
 
 # starting docker happens back in ../main.yml where it calls ../restart.yml
diff --git a/roles/openshift_node/tasks/install.yml b/roles/openshift_node/tasks/install.yml
index 9a91e2fb6..1ed4a05c1 100644
--- a/roles/openshift_node/tasks/install.yml
+++ b/roles/openshift_node/tasks/install.yml
@@ -5,6 +5,8 @@
     package:
       name: "{{ openshift.common.service_type }}-node{{ (openshift_pkg_version | default('')) | oo_image_tag_to_rpm_version(include_dash=True) }}"
       state: present
+    register: result
+    until: result | success
 
   - name: Install sdn-ovs package
     package:
@@ -12,11 +14,15 @@
       state: present
     when:
     - openshift_node_use_openshift_sdn | bool
+    register: result
+    until: result | success
 
   - name: Install conntrack-tools package
     package:
       name: "conntrack-tools"
       state: present
+    register: result
+    until: result | success
 
 - when:
   - openshift.common.is_containerized | bool
diff --git a/roles/openshift_node/tasks/storage_plugins/ceph.yml b/roles/openshift_node/tasks/storage_plugins/ceph.yml
index 037efe81a..72a3b837f 100644
--- a/roles/openshift_node/tasks/storage_plugins/ceph.yml
+++ b/roles/openshift_node/tasks/storage_plugins/ceph.yml
@@ -2,3 +2,5 @@
 - name: Install Ceph storage plugin dependencies
   package: name=ceph-common state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
index 1b8a7ad50..08ea71a0c 100644
--- a/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/glusterfs.yml
@@ -2,6 +2,8 @@
 - name: Install GlusterFS storage plugin dependencies
   package: name=glusterfs-fuse state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of fusefs sebooleans
   command: getsebool {{ item }}
diff --git a/roles/openshift_node/tasks/storage_plugins/iscsi.yml b/roles/openshift_node/tasks/storage_plugins/iscsi.yml
index 1c5478c55..ece68dc71 100644
--- a/roles/openshift_node/tasks/storage_plugins/iscsi.yml
+++ b/roles/openshift_node/tasks/storage_plugins/iscsi.yml
@@ -2,3 +2,5 @@
 - name: Install iSCSI storage plugin dependencies
   package: name=iscsi-initiator-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
diff --git a/roles/openshift_node/tasks/storage_plugins/nfs.yml b/roles/openshift_node/tasks/storage_plugins/nfs.yml
index 7e1035893..5eacf42e8 100644
--- a/roles/openshift_node/tasks/storage_plugins/nfs.yml
+++ b/roles/openshift_node/tasks/storage_plugins/nfs.yml
@@ -2,6 +2,8 @@
 - name: Install NFS storage plugin dependencies
   package: name=nfs-utils state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Check for existence of nfs sebooleans
   command: getsebool {{ item }}
diff --git a/roles/openshift_node/tasks/upgrade.yml b/roles/openshift_node/tasks/upgrade.yml
index ff3478800..561b56918 100644
--- a/roles/openshift_node/tasks/upgrade.yml
+++ b/roles/openshift_node/tasks/upgrade.yml
@@ -107,6 +107,8 @@
     name: openvswitch
     state: latest
   when: not openshift.common.is_containerized | bool
+  register: result
+  until: result | success
 
 - name: Update oreg value
   yedit:
diff --git a/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
index a998acf21..fcbe1a598 100644
--- a/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
+++ b/roles/openshift_node/tasks/upgrade/rpm_upgrade.yml
@@ -8,10 +8,14 @@
 # We verified latest rpm available is suitable, so just yum update.
 - name: Upgrade packages
   package: "name={{ openshift.common.service_type }}-{{ component }}{{ openshift_pkg_version }} state=present"
+  register: result
+  until: result | success
 
 - name: Ensure python-yaml present for config upgrade
   package: name=PyYAML state=present
   when: not openshift.common.is_atomic | bool
+  register: result
+  until: result | success
 
 - name: Install Node service file
   template: