Allow storage migrations to be optional and/or non fatal

Allow configuration of these options, these are their defaults
openshift_upgrade_pre_storage_migration_enabled=true
openshift_upgrade_post_storage_migration_enabled=true
openshift_upgrade_pre_storage_migration_fatal=true
openshift_upgrade_post_storage_migration_fatal=false

3 files changed, 41 insertions, 8 deletions

diff --git a/inventory/byo/hosts.origin.example b/inventory/byo/hosts.origin.example
index ff2bdb0c5..de7493f71 100644
--- a/inventory/byo/hosts.origin.example
+++ b/inventory/byo/hosts.origin.example
@@ -883,6 +883,14 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # openshift_upgrade_nodes_serial=4 openshift_upgrade_nodes_max_fail_percentage=49
 # where as this would not
 # openshift_upgrade_nodes_serial=4 openshift_upgrade_nodes_max_fail_percentage=50
+#
+# Multiple data migrations take place and if they fail they will fail the upgrade
+# You may wish to disable these or make them non fatal
+#
+# openshift_upgrade_pre_storage_migration_enabled=true
+# openshift_upgrade_pre_storage_migration_fatal==true
+# openshift_upgrade_post_storage_migration_enabled=true
+# openshift_upgrade_post_storage_migration_fatal==false
 
 # host group for masters
 [masters]
diff --git a/inventory/byo/hosts.ose.example b/inventory/byo/hosts.ose.example
index 68639eebf..62a364e0d 100644
--- a/inventory/byo/hosts.ose.example
+++ b/inventory/byo/hosts.ose.example
@@ -879,6 +879,14 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
 # openshift_upgrade_nodes_serial=4 openshift_upgrade_nodes_max_fail_percentage=49
 # where as this would not
 # openshift_upgrade_nodes_serial=4 openshift_upgrade_nodes_max_fail_percentage=50
+#
+# Multiple data migrations take place and if they fail they will fail the upgrade
+# You may wish to disable these or make them non fatal
+#
+# openshift_upgrade_pre_storage_migration_enabled=true
+# openshift_upgrade_pre_storage_migration_fatal==true
+# openshift_upgrade_post_storage_migration_enabled=true
+# openshift_upgrade_post_storage_migration_fatal==false
 
 # host group for masters
 [masters]
diff --git a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
index 2b2f10aee..695dc3140 100644
--- a/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
+++ b/playbooks/common/openshift-cluster/upgrades/upgrade_control_plane.yml
@@ -12,6 +12,12 @@
     command: >
       {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
       migrate storage --include=* --confirm
+    register: l_pb_upgrade_control_plane_pre_upgrade_storage
+    when: openshift_upgrade_pre_storage_migration_enabled | default(true,true) | bool
+    failed_when:
+    - openshift_upgrade_pre_storage_migration_enabled | default(true,true) | bool
+    - l_pb_upgrade_control_plane_pre_upgrade_storage.rc != 0
+    - openshift_upgrade_pre_storage_migration_fatal | default(true,true) | bool
 
 # If facts cache were for some reason deleted, this fact may not be set, and if not set
 # it will always default to true. This causes problems for the etcd data dir fact detection
@@ -140,16 +146,21 @@
   - include: "{{ openshift_master_upgrade_post_hook }}"
     when: openshift_master_upgrade_post_hook is defined
 
-  - set_fact:
-      master_update_complete: True
-
-- name: Post master upgrade - Upgrade clusterpolicies storage
-  hosts: oo_first_master
-  tasks:
-  - name: Upgrade clusterpolicies storage
+  - name: Post master upgrade - Upgrade clusterpolicies storage
     command: >
       {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
       migrate storage --include=clusterpolicies --confirm
+    register: l_pb_upgrade_control_plane_post_upgrade_storage
+    when: openshift_upgrade_post_storage_migration_enabled | default(true,true) | bool
+    failed_when:
+    - openshift_upgrade_post_storage_migration_enabled | default(true,true) | bool
+    - l_pb_upgrade_control_plane_post_upgrade_storage.rc != 0
+    - openshift_upgrade_post_storage_migration_fatal | default(false,true) | bool
+    run_once: true
+    delegate_to: oo_first_master
+
+  - set_fact:
+      master_update_complete: True
 
 ##############################################################################
 # Gate on master update complete
@@ -230,11 +241,17 @@
     - reconcile_scc_result.rc == 0
     run_once: true
 
-  - name: Upgrade job storage
+  - name: Migrate storage post policy reconciliation
     command: >
       {{ openshift.common.client_binary }} adm --config={{ openshift.common.config_base }}/master/admin.kubeconfig
       migrate storage --include=* --confirm
     run_once: true
+    register: l_pb_upgrade_control_plane_post_upgrade_storage
+    when: openshift_upgrade_post_storage_migration_enabled | default(true,true) | bool
+    failed_when:
+    - openshift_upgrade_post_storage_migration_enabled | default(true,true) | bool
+    - l_pb_upgrade_control_plane_post_upgrade_storage.rc != 0
+    - openshift_upgrade_post_storage_migration_fatal | default(false,true) | bool
 
   - set_fact:
       reconcile_complete: True