Contiv multi-master and other fixes

Contiv's etcd was not being deployed correctly when using more than
one master.  To make it easier to manage, it has been moved into a
k8s container.

The api proxy was hardcoded to an old version (1.1.1), and in some
environments would run into a docker error.  This has been moved into
a k8s container for easier management.

The firewall was too permissive on several ports.  Many were open to
the world when they should have only been accessible inside the
cluster.

Many of the contiv role variables were not prefixed with 'contiv',
which may end up clobbering variables from another role.  Now all the
contiv specific role variables start with 'contiv_'.

The api proxy's default self-signed certificate was bundled with the
role.  This means someone with read-only MITM access and this key
could decrypt traffic.  Granted a user defined certificate from a
trusted CA should be used in a production environment, it is still
better to generate one in each environment when one is not provided.

18 files changed, 505 insertions, 207 deletions

diff --git a/roles/contiv/tasks/aci.yml b/roles/contiv/tasks/aci.yml
index 30d2eb339..8a56b3590 100644
--- a/roles/contiv/tasks/aci.yml
+++ b/roles/contiv/tasks/aci.yml
@@ -11,7 +11,7 @@
 - name: ACI | Copy shell script used by aci-gw service
   template:
     src: aci_gw.j2
-    dest: "{{ bin_dir }}/aci_gw.sh"
+    dest: "{{ contiv_bin_dir }}/aci_gw.sh"
     mode: u=rwx,g=rx,o=rx
 
 - name: ACI | Copy systemd units for aci-gw
diff --git a/roles/contiv/tasks/api_proxy.yml b/roles/contiv/tasks/api_proxy.yml
new file mode 100644
index 000000000..8b524dd6e
--- /dev/null
+++ b/roles/contiv/tasks/api_proxy.yml
@@ -0,0 +1,120 @@
+---
+- name: API proxy | Create contiv-api-proxy openshift user
+  oc_serviceaccount:
+    state: present
+    name: contiv-api-proxy
+    namespace: kube-system
+  run_once: true
+
+- name: API proxy | Set contiv-api-proxy openshift user permissions
+  oc_adm_policy_user:
+    user: system:serviceaccount:kube-system:contiv-api-proxy
+    resource_kind: scc
+    resource_name: hostnetwork
+    state: present
+  run_once: true
+
+- name: API proxy | Create temp directory for doing work
+  command: mktemp -d /tmp/openshift-contiv-XXXXXX
+  register: mktemp
+  changed_when: False
+  # For things that pass temp files between steps, we want to make sure they
+  # run on the same node.
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Check for existing api proxy secret volume
+  oc_obj:
+    namespace: kube-system
+    kind: secret
+    state: list
+    selector: "name=contiv-api-proxy-secret"
+  register: existing_secret_volume
+  run_once: true
+
+- name: API proxy | Generate a self signed certificate for api proxy
+  command: openssl req -new -nodes -x509 -subj "/C=US/ST=/L=/O=/CN=localhost" -days 3650 -keyout "{{ mktemp.stdout }}/key.pem" -out "{{ mktemp.stdout }}/cert.pem" -extensions v3_ca
+  when: (contiv_api_proxy_cert is not defined or contiv_api_proxy_key is not defined)
+        and not existing_secret_volume.results.results[0]['items']
+  register: created_self_signed_cert
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Read self signed certificate file
+  command: cat "{{ mktemp.stdout }}/cert.pem"
+  register: generated_cert
+  when: created_self_signed_cert.changed
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Read self signed key file
+  command: cat "{{ mktemp.stdout }}/key.pem"
+  register: generated_key
+  when: created_self_signed_cert.changed
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Create api-proxy-secrets.yml from template using generated cert
+  template:
+    src: api-proxy-secrets.yml.j2
+    dest: "{{ mktemp.stdout }}/api-proxy-secrets.yml"
+  vars:
+    key: "{{ generated_key.stdout }}"
+    cert: "{{ generated_cert.stdout }}"
+  when: created_self_signed_cert.changed
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Create api-proxy-secrets.yml from template using user defined cert
+  template:
+    src: api-proxy-secrets.yml.j2
+    dest: "{{ mktemp.stdout }}/api-proxy-secrets.yml"
+  vars:
+    key: "{{ lookup('file', contiv_api_proxy_key) }}"
+    cert: "{{ lookup('file', contiv_api_proxy_cert) }}"
+  when: contiv_api_proxy_cert is defined and contiv_api_proxy_key is defined
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Create secret certificate volume
+  oc_obj:
+    state: present
+    namespace: "kube-system"
+    kind: secret
+    name: contiv-api-proxy-secret
+    files:
+      - "{{ mktemp.stdout }}/api-proxy-secrets.yml"
+  when: (contiv_api_proxy_cert is defined and contiv_api_proxy_key is defined)
+        or created_self_signed_cert.changed
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Create api-proxy-daemonset.yml from template
+  template:
+    src: api-proxy-daemonset.yml.j2
+    dest: "{{ mktemp.stdout }}/api-proxy-daemonset.yml"
+  vars:
+    etcd_host: "etcd://{{ groups.oo_etcd_to_config.0 }}:{{ contiv_etcd_port }}"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+# Always "import" this file, k8s won't do anything if it matches exactly what
+# is already in the cluster.
+- name: API proxy | Add API proxy daemonset
+  oc_obj:
+    state: present
+    namespace: "kube-system"
+    kind: daemonset
+    name: contiv-api-proxy
+    files:
+      - "{{ mktemp.stdout }}/api-proxy-daemonset.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: API proxy | Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
diff --git a/roles/contiv/tasks/default_network.yml b/roles/contiv/tasks/default_network.yml
index 8a928ea54..e9763d34a 100644
--- a/roles/contiv/tasks/default_network.yml
+++ b/roles/contiv/tasks/default_network.yml
@@ -1,71 +1,71 @@
 ---
-- name: Contiv | Wait for netmaster
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" tenant ls'
+- name: Default network | Wait for netmaster
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" tenant ls'
   register: tenant_result
   until: tenant_result.stdout.find("default") != -1
   retries: 9
   delay: 10
 
-- name: Contiv | Set globals
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" global set --fabric-mode {{ contiv_fabric_mode }} --vlan-range {{ contiv_vlan_range }} --fwd-mode {{ netplugin_fwd_mode }} --private-subnet {{ contiv_private_ext_subnet }}'
+- name: Default network | Set globals
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" global set --fabric-mode {{ contiv_fabric_mode }} --vlan-range {{ contiv_vlan_range }} --fwd-mode {{ contiv_netplugin_fwd_mode }} --private-subnet {{ contiv_private_ext_subnet }}'
   run_once: true
 
-- name: Contiv | Set arp mode to flood if ACI
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" global set --arp-mode flood'
+- name: Default network | Set arp mode to flood if ACI
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" global set --arp-mode flood'
   when: contiv_fabric_mode == "aci"
   run_once: true
 
-- name: Contiv | Check if default-net exists
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" net ls'
+- name: Default network | Check if default-net exists
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" net ls'
   register: net_result
   run_once: true
 
-- name: Contiv | Create default-net
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" net create --subnet={{ contiv_default_subnet }} -e {{ contiv_encap_mode }} -p {{ contiv_default_network_tag }} --gateway {{ contiv_default_gw }} default-net'
+- name: Default network | Create default-net
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" net create --subnet={{ contiv_default_subnet }} -e {{ contiv_encap_mode }} -p {{ contiv_default_network_tag }} --gateway {{ contiv_default_gw }} default-net'
   when: net_result.stdout.find("default-net") == -1
   run_once: true
 
-- name: Contiv | Create host access infra network for VxLan routing case
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" net create --subnet={{ contiv_h1_subnet_default }} --gateway={{ contiv_h1_gw_default }} --nw-type="infra" contivh1'
-  when: (contiv_encap_mode == "vxlan") and (netplugin_fwd_mode == "routing")
+- name: Default network | Create host access infra network for VxLan routing case
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" net create --subnet={{ contiv_h1_subnet_default }} --gateway={{ contiv_h1_gw_default }} --nw-type="infra" contivh1'
+  when: (contiv_encap_mode == "vxlan") and (contiv_netplugin_fwd_mode == "routing")
   run_once: true
 
-#- name: Contiv | Create an allow-all policy for the default-group
-#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" policy create ose-allow-all-policy'
+#- name: Default network | Create an allow-all policy for the default-group
+#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" policy create ose-allow-all-policy'
 #  when: contiv_fabric_mode == "aci"
 #  run_once: true
 
-- name: Contiv | Set up aci external contract to consume default external contract
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" external-contracts create -c -a {{ apic_default_external_contract }} oseExtToConsume'
+- name: Default network | Set up aci external contract to consume default external contract
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" external-contracts create -c -a {{ contiv_apic_default_external_contract }} oseExtToConsume'
   when: (contiv_fabric_mode == "aci") and (apic_configure_default_policy == true)
   run_once: true
 
-- name: Contiv | Set up aci external contract to provide default external contract
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" external-contracts create -p -a {{ apic_default_external_contract }} oseExtToProvide'
+- name: Default network | Set up aci external contract to provide default external contract
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" external-contracts create -p -a {{ contiv_apic_default_external_contract }} oseExtToProvide'
   when: (contiv_fabric_mode == "aci") and (apic_configure_default_policy == true)
   run_once: true
 
-- name: Contiv | Create aci default-group
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" group create default-net default-group'
+- name: Default network | Create aci default-group
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" group create default-net default-group'
   when: contiv_fabric_mode == "aci"
   run_once: true
 
-- name: Contiv | Add external contracts to the default-group
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" group create -e oseExtToConsume -e oseExtToProvide default-net default-group'
+- name: Default network | Add external contracts to the default-group
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" group create -e oseExtToConsume -e oseExtToProvide default-net default-group'
   when: (contiv_fabric_mode == "aci") and (apic_configure_default_policy == true)
   run_once: true
 
-#- name: Contiv | Add policy rule 1 for allow-all policy
-#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" policy rule-add -d in --action allow ose-allow-all-policy 1'
+#- name: Default network | Add policy rule 1 for allow-all policy
+#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" policy rule-add -d in --action allow ose-allow-all-policy 1'
 #  when: contiv_fabric_mode == "aci"
 #  run_once: true
 
-#- name: Contiv | Add policy rule 2 for allow-all policy
-#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" policy rule-add -d out --action allow ose-allow-all-policy 2'
+#- name: Default network | Add policy rule 2 for allow-all policy
+#  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" policy rule-add -d out --action allow ose-allow-all-policy 2'
 #  when: contiv_fabric_mode == "aci"
 #  run_once: true
 
-- name: Contiv | Create default aci app profile
-  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ netmaster_port }}" app-profile create -g default-group {{ apic_default_app_profile }}'
+- name: Default network | Create default aci app profile
+  command: 'netctl --netmaster "http://{{ inventory_hostname }}:{{ contiv_netmaster_port }}" app-profile create -g default-group {{ contiv_apic_default_app_profile }}'
   when: contiv_fabric_mode == "aci"
   run_once: true
diff --git a/roles/contiv/tasks/download_bins.yml b/roles/contiv/tasks/download_bins.yml
index 831fd360a..47d74da9c 100644
--- a/roles/contiv/tasks/download_bins.yml
+++ b/roles/contiv/tasks/download_bins.yml
@@ -4,7 +4,7 @@
     path: "{{ contiv_current_release_directory }}"
     state: directory
 
-- name: Install bzip2
+- name: Download Bins | Install bzip2
   yum:
     name: bzip2
     state: installed
@@ -18,9 +18,9 @@
     mode: 0755
     validate_certs: False
   environment:
-    http_proxy: "{{ http_proxy|default('') }}"
-    https_proxy: "{{ https_proxy|default('') }}"
-    no_proxy: "{{ no_proxy|default('') }}"
+    http_proxy: "{{ contiv_http_proxy|default('') }}"
+    https_proxy: "{{ contiv_https_proxy|default('') }}"
+    no_proxy: "{{ contiv_no_proxy|default('') }}"
 
 - name: Download Bins | Extract Contiv tar file
   unarchive:
@@ -30,19 +30,19 @@
 
 - name: Download Bins | Download cni tar file
   get_url:
-    url: "{{ cni_bin_url }}"
-    dest: "{{ cni_download_dir }}"
+    url: "{{ contiv_cni_bin_url }}"
+    dest: "{{ contiv_cni_download_dir }}"
     mode: 0755
     validate_certs: False
   environment:
-    http_proxy: "{{ http_proxy|default('') }}"
-    https_proxy: "{{ https_proxy|default('') }}"
-    no_proxy: "{{ no_proxy|default('') }}"
+    http_proxy: "{{ contiv_http_proxy|default('') }}"
+    https_proxy: "{{ contiv_https_proxy|default('') }}"
+    no_proxy: "{{ contiv_no_proxy|default('') }}"
   register: download_file
 
 - name: Download Bins | Extract cni tar file
   unarchive:
     src: "{{ download_file.dest }}"
-    dest: "{{ cni_download_dir }}"
+    dest: "{{ contiv_cni_download_dir }}"
     copy: no
   when: download_file.changed
diff --git a/roles/contiv/tasks/etcd.yml b/roles/contiv/tasks/etcd.yml
new file mode 100644
index 000000000..b08ead982
--- /dev/null
+++ b/roles/contiv/tasks/etcd.yml
@@ -0,0 +1,114 @@
+---
+# To run contiv-etcd in a container as non-root, we need to match the uid/gid
+# with the filesystem permissions on the host.
+- name: Contiv etcd | Create local unix group
+  group:
+    name: "{{ contiv_etcd_system_group }}"
+    gid: "{{ contiv_etcd_system_gid }}"
+    system: yes
+
+- name: Contiv etcd | Create local unix user
+  user:
+    name: "{{ contiv_etcd_system_user }}"
+    createhome: no
+    uid: "{{ contiv_etcd_system_uid }}"
+    group: "{{ contiv_etcd_system_group }}"
+    home: "{{ contiv_etcd_data_dir }}"
+    shell: /bin/false
+    system: yes
+
+- name: Contiv etcd | Create directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: g-rwx,o-rwx
+    owner: "{{ contiv_etcd_system_user }}"
+    group: "{{ contiv_etcd_system_group }}"
+    setype: svirt_sandbox_file_t
+    seuser: system_u
+    serole: object_r
+    selevel: s0
+    recurse: yes
+  with_items:
+    - "{{ contiv_etcd_data_dir }}"
+    - "{{ contiv_etcd_conf_dir }}"
+
+- name: Contiv etcd | Create contiv-etcd openshift user
+  oc_serviceaccount:
+    state: present
+    name: contiv-etcd
+    namespace: kube-system
+  run_once: true
+
+- name: Contiv etcd | Create temp directory for doing work
+  command: mktemp -d /tmp/openshift-contiv-XXXXXX
+  register: mktemp
+  changed_when: False
+  # For things that pass temp files between steps, we want to make sure they
+  # run on the same node.
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Create etcd-scc.yml from template
+  template:
+    src: etcd-scc.yml.j2
+    dest: "{{ mktemp.stdout }}/etcd-scc.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Create etcd.yml from template
+  template:
+    src: etcd-daemonset.yml.j2
+    dest: "{{ mktemp.stdout }}/etcd-daemonset.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Create etcd-proxy.yml from template
+  template:
+    src: etcd-proxy-daemonset.yml.j2
+    dest: "{{ mktemp.stdout }}/etcd-proxy-daemonset.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Add etcd scc
+  oc_obj:
+    state: present
+    namespace: "kube-system"
+    kind: SecurityContextConstraints
+    name: contiv-etcd
+    files:
+      - "{{ mktemp.stdout }}/etcd-scc.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+# Always "import" this file, k8s won't do anything if it matches exactly what
+# is already in the cluster.
+- name: Contiv etcd | Add etcd daemonset
+  oc_obj:
+    state: present
+    namespace: "kube-system"
+    kind: daemonset
+    name: contiv-etcd
+    files:
+      - "{{ mktemp.stdout }}/etcd-daemonset.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Add etcd-proxy daemonset
+  oc_obj:
+    state: present
+    namespace: "kube-system"
+    kind: daemonset
+    name: contiv-etcd-proxy
+    files:
+      - "{{ mktemp.stdout }}/etcd-proxy-daemonset.yml"
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
+
+- name: Contiv etcd | Delete temp directory
+  file:
+    name: "{{ mktemp.stdout }}"
+    state: absent
+  changed_when: False
+  delegate_to: "{{ groups.oo_masters_to_config.0 }}"
+  run_once: true
diff --git a/roles/contiv/tasks/main.yml b/roles/contiv/tasks/main.yml
index cb9196a71..4d530ae90 100644
--- a/roles/contiv/tasks/main.yml
+++ b/roles/contiv/tasks/main.yml
@@ -1,14 +1,15 @@
 ---
-- name: Ensure bin_dir exists
+- include_tasks: old_version_cleanup.yml
+
+- name: Ensure contiv_bin_dir exists
   file:
-    path: "{{ bin_dir }}"
+    path: "{{ contiv_bin_dir }}"
     recurse: yes
     state: directory
 
 - include_tasks: download_bins.yml
 
 - include_tasks: netmaster.yml
-  when: contiv_role == "netmaster"
+  when: contiv_master
 
 - include_tasks: netplugin.yml
-  when: contiv_role == "netplugin"
diff --git a/roles/contiv/tasks/netmaster.yml b/roles/contiv/tasks/netmaster.yml
index 6f15af8c2..83203bc2e 100644
--- a/roles/contiv/tasks/netmaster.yml
+++ b/roles/contiv/tasks/netmaster.yml
@@ -1,34 +1,16 @@
 ---
 - include_tasks: netmaster_firewalld.yml
-  when: has_firewalld
+  when: contiv_has_firewalld
 
 - include_tasks: netmaster_iptables.yml
-  when: not has_firewalld and has_iptables
+  when: not contiv_has_firewalld and contiv_has_iptables
 
-- name: Netmaster | Check is /etc/hosts file exists
-  stat:
-    path: /etc/hosts
-  register: hosts
-
-- name: Netmaster | Create hosts file if it is not present
-  file:
-    path: /etc/hosts
-    state: touch
-  when: not hosts.stat.exists
-
-- name: Netmaster | Build hosts file
-  lineinfile:
-    dest: /etc/hosts
-    regexp: .*netmaster$
-    line: "{{ hostvars[item]['ansible_' + netmaster_interface].ipv4.address }} netmaster"
-    state: present
-  when: hostvars[item]['ansible_' + netmaster_interface].ipv4.address is defined
-  with_items: "{{ groups['masters'] }}"
+- include_tasks: etcd.yml
 
 - name: Netmaster | Create netmaster symlinks
   file:
     src: "{{ contiv_current_release_directory }}/{{ item }}"
-    dest: "{{ bin_dir }}/{{ item }}"
+    dest: "{{ contiv_bin_dir }}/{{ item }}"
     state: link
   with_items:
     - netmaster
@@ -75,3 +57,5 @@
 
 - include_tasks: default_network.yml
   when: contiv_default_network == true
+
+- include_tasks: api_proxy.yml
diff --git a/roles/contiv/tasks/netmaster_firewalld.yml b/roles/contiv/tasks/netmaster_firewalld.yml
index 2975351ac..0d52f821d 100644
--- a/roles/contiv/tasks/netmaster_firewalld.yml
+++ b/roles/contiv/tasks/netmaster_firewalld.yml
@@ -1,16 +1,17 @@
 ---
-- name: Netmaster Firewalld | Open Netmaster port
+- name: Netmaster Firewalld | Add internal rules
   firewalld:
-    port: "{{ netmaster_port }}/tcp"
-    permanent: false
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
+    immediate: true
+    permanent: true
+    port: "{{ item[0] }}"
+    source: "{{ item[1] }}"
+  with_nested:
+    - "{{ contiv_netmaster_internal }}"
+    - "{{ groups.oo_nodes_to_config|difference(hostvars[inventory_hostname]['ansible_' + contiv_netmaster_interface].ipv4.address)|list }}"
 
-- name: Netmaster Firewalld | Save Netmaster port
+- name: Netmaster Firewalld | Add external rules
   firewalld:
-    port: "{{ netmaster_port }}/tcp"
+    immediate: true
     permanent: true
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
+    port: "{{ item }}"
+  with_items: "{{ contiv_netmaster_external }}"
diff --git a/roles/contiv/tasks/netmaster_iptables.yml b/roles/contiv/tasks/netmaster_iptables.yml
index c98e7b6a5..3b68ea0c3 100644
--- a/roles/contiv/tasks/netmaster_iptables.yml
+++ b/roles/contiv/tasks/netmaster_iptables.yml
@@ -1,27 +1,32 @@
 ---
-- name: Netmaster IPtables | Get iptables rules
-  command: iptables -L --wait
-  register: iptablesrules
-  check_mode: no
-
-- name: Netmaster IPtables | Enable iptables at boot
-  service:
-    name: iptables
-    enabled: yes
-    state: started
-
-- name: Netmaster IPtables | Open Netmaster with iptables
-  command: /sbin/iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "contiv"
-  with_items:
-    - "{{ contiv_rpc_port1 }}"
-    - "{{ contiv_rpc_port2 }}"
-    - "{{ contiv_rpc_port3 }}"
-  when: iptablesrules.stdout.find("contiv") == -1
+- name: Netmaster IPtables | Add internal rules
+  iptables:
+    action: insert
+    chain: INPUT
+    # Parsed from the contiv_netmaster_internal list, this will be tcp or udp.
+    protocol: "{{ item[0].split('/')[1] }}"
+    match: "{{ item[0].split('/')[1] }}"
+    # Parsed from the contiv_netmaster_internal list, this will be a port number.
+    destination_port: "{{ item[0].split('/')[0] }}"
+    # This is an IP address from a node in the cluster.
+    source: "{{ item[1] }}"
+    jump: ACCEPT
+    comment: contiv
+  with_nested:
+    - "{{ contiv_netmaster_internal }}"
+    - "{{ groups.oo_nodes_to_config|difference(hostvars[inventory_hostname]['ansible_' + contiv_netmaster_interface].ipv4.address)|list }}"
   notify: Save iptables rules
 
-- name: Netmaster IPtables | Open netmaster main port
-  command: /sbin/iptables -I INPUT 1 -p tcp -s {{ item }} --dport {{ netmaster_port }} -j ACCEPT -m comment --comment "contiv"
-  with_items:
-    - "{{ groups.oo_nodes_to_config|difference(hostvars[inventory_hostname]['ansible_' + netmaster_interface].ipv4.address)|list }}"
-  when: iptablesrules.stdout.find("contiv") == -1
+- name: Netmaster IPtables | Add external rules
+  iptables:
+    action: insert
+    chain: INPUT
+    # Parsed from the contiv_netmaster_external list, this will be tcp or udp.
+    protocol: "{{ item.split('/')[1] }}"
+    match: "{{ item.split('/')[1] }}"
+    # Parsed from the contiv_netmaster_external list, this will be a port number.
+    destination_port: "{{ item.split('/')[0] }}"
+    jump: ACCEPT
+    comment: contiv
+  with_items: "{{ contiv_netmaster_external }}"
   notify: Save iptables rules
diff --git a/roles/contiv/tasks/netplugin.yml b/roles/contiv/tasks/netplugin.yml
index 540f6e4bc..60f432202 100644
--- a/roles/contiv/tasks/netplugin.yml
+++ b/roles/contiv/tasks/netplugin.yml
@@ -1,9 +1,9 @@
 ---
 - include_tasks: netplugin_firewalld.yml
-  when: has_firewalld
+  when: contiv_has_firewalld
 
 - include_tasks: netplugin_iptables.yml
-  when: has_iptables
+  when: not contiv_has_firewalld and contiv_has_iptables
 
 - name: Netplugin | Ensure localhost entry correct in /etc/hosts
   lineinfile:
@@ -20,41 +20,40 @@
     state: absent
 
 - include_tasks: ovs.yml
-  when: netplugin_driver == "ovs"
+  when: contiv_netplugin_driver == "ovs"
 
 - name: Netplugin | Create Netplugin bin symlink
   file:
     src: "{{ contiv_current_release_directory }}/netplugin"
-    dest: "{{ bin_dir }}/netplugin"
+    dest: "{{ contiv_bin_dir }}/netplugin"
     state: link
 
-
-- name: Netplugin | Ensure cni_bin_dir exists
+- name: Netplugin | Ensure contiv_cni_bin_dir exists
   file:
-    path: "{{ cni_bin_dir }}"
+    path: "{{ contiv_cni_bin_dir }}"
     recurse: yes
     state: directory
 
 - name: Netplugin | Create CNI bin symlink
   file:
     src: "{{ contiv_current_release_directory }}/contivk8s"
-    dest: "{{ cni_bin_dir }}/contivk8s"
+    dest: "{{ contiv_cni_bin_dir }}/contivk8s"
     state: link
 
 - name: Netplugin | Copy CNI loopback bin
   copy:
-    src: "{{ cni_download_dir }}/loopback"
-    dest: "{{ cni_bin_dir }}/loopback"
+    src: "{{ contiv_cni_download_dir }}/loopback"
+    dest: "{{ contiv_cni_bin_dir }}/loopback"
     remote_src: True
     mode: 0755
 
-- name: Netplugin | Ensure kube_plugin_dir and cni/net.d directories exist
+- name: Netplugin | Ensure contiv_kube_plugin_dir and cni/net.d directories exist
   file:
     path: "{{ item }}"
     recurse: yes
     state: directory
   with_items:
-    - "{{ kube_plugin_dir }}"
+    - "{{ contiv_kube_plugin_dir }}"
     - "/etc/cni/net.d"
 
 - name: Netplugin | Ensure contiv_config_dir exists
@@ -68,7 +67,7 @@
     src: contiv_cni.conf
     dest: "{{ item }}"
   with_items:
-    - "{{ kube_plugin_dir }}/contiv_cni.conf"
+    - "{{ contiv_kube_plugin_dir }}/contiv_cni.conf"
     - "/etc/cni/net.d"
 # notify: restart kubelet
 
@@ -85,11 +84,11 @@
     mode: 0644
   notify: restart netplugin
 
-- name: Docker | Make sure proxy setting exists
+- name: Netplugin | Make sure docker proxy setting exists
   lineinfile:
     dest: /etc/sysconfig/docker-network
     regexp: '^https_proxy.*'
-    line: 'https_proxy={{ https_proxy }}'
+    line: 'https_proxy={{ contiv_https_proxy }}'
     state: present
   register: docker_updated
 
@@ -103,9 +102,9 @@
   command: systemctl daemon-reload
   when: docker_updated is changed
 
-- name: Docker | Restart docker
+- name: Netplugin | Restart docker
   service:
-    name: "{{ openshift_docker_service_name }}"
+    name: "{{ contiv_openshift_docker_service_name }}"
     state: restarted
   when: docker_updated is changed
   register: l_docker_restart_docker_in_contiv_result
diff --git a/roles/contiv/tasks/netplugin_firewalld.yml b/roles/contiv/tasks/netplugin_firewalld.yml
index 3aeffae56..5ac531ec6 100644
--- a/roles/contiv/tasks/netplugin_firewalld.yml
+++ b/roles/contiv/tasks/netplugin_firewalld.yml
@@ -1,34 +1,17 @@
 ---
-- name: Netplugin Firewalld | Open Netplugin port
+- name: Netplugin Firewalld | Add internal rules
   firewalld:
-    port: "{{ netplugin_port }}/tcp"
-    permanent: false
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
-
-- name: Netplugin Firewalld | Save Netplugin port
-  firewalld:
-    port: "{{ netplugin_port }}/tcp"
+    immediate: true
     permanent: true
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
-
-- name: Netplugin Firewalld | Open vxlan port
-  firewalld:
-    port: "8472/udp"
-    permanent: false
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
-  when: contiv_encap_mode == "vxlan"
+    port: "{{ item[0] }}"
+    source: "{{ item[1] }}"
+  with_nested:
+    - "{{ contiv_netplugin_internal }}"
+    - "{{ groups.oo_nodes_to_config|difference(hostvars[inventory_hostname]['ansible_' + contiv_netmaster_interface].ipv4.address)|list }}"
 
-- name: Netplugin Firewalld | Save firewalld vxlan port for flanneld
+- name: Netplugin Firewalld | Add dns rule
   firewalld:
-    port: "8472/udp"
+    immediate: true
     permanent: true
-    state: enabled
-  # in case this is also a node where firewalld turned off
-  ignore_errors: yes
-  when: contiv_encap_mode == "vxlan"
+    port: "53/udp"
+    interface: contivh0
diff --git a/roles/contiv/tasks/netplugin_iptables.yml b/roles/contiv/tasks/netplugin_iptables.yml
index 3ea34645d..9d376f4e5 100644
--- a/roles/contiv/tasks/netplugin_iptables.yml
+++ b/roles/contiv/tasks/netplugin_iptables.yml
@@ -1,58 +1,52 @@
 ---
-- name: Netplugin IPtables | Get iptables rules
-  command: iptables -L --wait
-  register: iptablesrules
-  check_mode: no
+- name: Netplugin IPtables | Add internal rules
+  iptables:
+    action: insert
+    chain: INPUT
+    protocol: "{{ item[0].split('/')[1] }}"
+    match: "{{ item[0].split('/')[1] }}"
+    destination_port: "{{ item[0].split('/')[0] }}"
+    source: "{{ item[1] }}"
+    jump: ACCEPT
+    comment: contiv
+  with_nested:
+    - "{{ contiv_netplugin_internal }}"
+    - "{{ groups.oo_nodes_to_config|difference(hostvars[inventory_hostname]['ansible_' + contiv_netmaster_interface].ipv4.address)|list }}"
+  notify: Save iptables rules
+
+- name: Netplugin IPtables | Add [in] forward rules
+  iptables:
+    action: insert
+    chain: FORWARD
+    in_interface: "{{ item }}"
+    jump: ACCEPT
+    comment: contiv
+  with_items: "{{ contiv_netplugin_forward_interfaces }}"
+  notify: Save iptables rules
+
+- name: Netplugin IPtables | Add [out] forward rules
+  iptables:
+    action: insert
+    chain: FORWARD
+    out_interface: "{{ item }}"
+    jump: ACCEPT
+    comment: contiv
+  with_items: "{{ contiv_netplugin_forward_interfaces }}"
+  notify: Save iptables rules
+
+- name: Netplugin IPtables | Add dns rule
+  iptables:
+    action: insert
+    chain: INPUT
+    protocol: udp
+    match: udp
+    destination_port: 53
+    in_interface: contivh0
+    jump: ACCEPT
+    comment: contiv
+  notify: Save iptables rules
 
 - name: Netplugin IPtables | Enable iptables at boot
   service:
     name: iptables
     enabled: yes
-    state: started
-
-- name: Netplugin IPtables | Open Netmaster with iptables
-  command: /sbin/iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "contiv"
-  with_items:
-  - "{{ netmaster_port }}"
-  - "{{ contiv_rpc_port1 }}"
-  - "{{ contiv_rpc_port2 }}"
-  - "{{ contiv_rpc_port3 }}"
-  - "{{ contiv_etcd_port }}"
-  - "{{ kube_master_api_port }}"
-  when: iptablesrules.stdout.find("contiv") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Open vxlan port with iptables
-  command: /sbin/iptables -I INPUT 1 -p udp --dport 8472 -j ACCEPT -m comment --comment "netplugin vxlan 8472"
-  when: iptablesrules.stdout.find("netplugin vxlan 8472") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Open vxlan port with iptables
-  command: /sbin/iptables -I INPUT 1 -p udp --dport 4789 -j ACCEPT -m comment --comment "netplugin vxlan 4789"
-  when: iptablesrules.stdout.find("netplugin vxlan 4789") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Allow from contivh0
-  command: /sbin/iptables -I FORWARD 1 -i contivh0 -j ACCEPT -m comment --comment "contivh0 FORWARD input"
-  when: iptablesrules.stdout.find("contivh0 FORWARD input") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Allow to contivh0
-  command: /sbin/iptables -I FORWARD 1 -o contivh0 -j ACCEPT -m comment --comment "contivh0 FORWARD output"
-  when: iptablesrules.stdout.find("contivh0 FORWARD output") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Allow from contivh1
-  command: /sbin/iptables -I FORWARD 1 -i contivh1 -j ACCEPT -m comment --comment "contivh1 FORWARD input"
-  when: iptablesrules.stdout.find("contivh1 FORWARD input") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Allow to contivh1
-  command: /sbin/iptables -I FORWARD 1 -o contivh1 -j ACCEPT -m comment --comment "contivh1 FORWARD output"
-  when: iptablesrules.stdout.find("contivh1 FORWARD output") == -1
-  notify: Save iptables rules
-
-- name: Netplugin IPtables | Allow dns
-  command: /sbin/iptables -I INPUT 1 -p udp --dport 53 -j ACCEPT -m comment --comment "contiv dns"
-  when: iptablesrules.stdout.find("contiv dns") == -1
-  notify: Save iptables rules
diff --git a/roles/contiv/tasks/old_version_cleanup.yml b/roles/contiv/tasks/old_version_cleanup.yml
new file mode 100644
index 000000000..8b3d88096
--- /dev/null
+++ b/roles/contiv/tasks/old_version_cleanup.yml
@@ -0,0 +1,43 @@
+---
+- name: Old version cleanup | Check if old auth proxy service exists
+  stat:
+    path: /etc/systemd/system/auth-proxy.service
+  register: auth_proxy_stat
+
+- name: Old version cleanup | Stop old auth proxy
+  service:
+    name: auth-proxy
+    enabled: no
+    state: stopped
+  when: auth_proxy_stat.stat.exists
+
+# Note(NB): The new containerized contiv-etcd service uses the same data
+# directory on the host, so etcd data is not lost.
+- name: Old version cleanup | Check if old contiv-etcd service exists
+  stat:
+    path: /etc/systemd/system/contiv-etcd.service
+  register: contiv_etcd_stat
+
+- name: Old version cleanup | Stop old contiv-etcd
+  service:
+    name: contiv-etcd
+    enabled: no
+    state: stopped
+  when: contiv_etcd_stat.stat.exists
+
+- name: Old version cleanup | Delete old files
+  file:
+    state: absent
+    path: "{{ item }}"
+  with_items:
+    - /etc/systemd/system/auth-proxy.service
+    - /var/contiv/certs
+    - /usr/bin/auth_proxy.sh
+    - /etc/systemd/system/contiv-etcd.service
+    - /etc/systemd/system/contiv-etcd.service.d
+
+- include_tasks: old_version_cleanup_iptables.yml
+  when: not contiv_has_firewalld and contiv_has_iptables
+
+- include_tasks: old_version_cleanup_firewalld.yml
+  when: contiv_has_firewalld
diff --git a/roles/contiv/tasks/old_version_cleanup_firewalld.yml b/roles/contiv/tasks/old_version_cleanup_firewalld.yml
new file mode 100644
index 000000000..675a6358a
--- /dev/null
+++ b/roles/contiv/tasks/old_version_cleanup_firewalld.yml
@@ -0,0 +1,11 @@
+---
+- name: Old version cleanup | Delete old firewalld rules
+  firewalld:
+    state: absent
+    immediate: true
+    permanent: true
+    port: "{{ item }}"
+  with_items:
+    - "9999/tcp"
+    - "6640/tcp"
+    - "8472/udp"
diff --git a/roles/contiv/tasks/old_version_cleanup_iptables.yml b/roles/contiv/tasks/old_version_cleanup_iptables.yml
new file mode 100644
index 000000000..513357606
--- /dev/null
+++ b/roles/contiv/tasks/old_version_cleanup_iptables.yml
@@ -0,0 +1,44 @@
+---
+- name: Old version cleanup | Delete old forward [in] iptables rules
+  iptables:
+    state: absent
+    chain: FORWARD
+    in_interface: "{{ item }}"
+    jump: ACCEPT
+    comment: "{{ item }} FORWARD input"
+  with_items:
+    - contivh0
+    - contivh1
+  notify: Save iptables rules
+
+- name: Old version cleanup | Delete old forward [out] iptables rules
+  iptables:
+    state: absent
+    chain: FORWARD
+    out_interface: "{{ item }}"
+    jump: ACCEPT
+    comment: "{{ item }} FORWARD output"
+  with_items:
+    - contivh0
+    - contivh1
+  notify: Save iptables rules
+
+- name: Old version cleanup | Delete old input iptables rules
+  iptables:
+    state: absent
+    chain: INPUT
+    protocol: "{{ item.split('/')[1] }}"
+    match: "{{ item.split('/')[1] }}"
+    destination_port: "{{ item.split('/')[0] }}"
+    comment: "{{ item.split('/')[2] }}"
+    jump: ACCEPT
+  with_items:
+    - "53/udp/contiv dns"
+    - "4789/udp/netplugin vxlan 4789"
+    - "8472/udp/netplugin vxlan 8472"
+    - "9003/tcp/contiv"
+    - "9002/tcp/contiv"
+    - "9001/tcp/contiv"
+    - "9999/tcp/contiv"
+    - "10000/tcp/Contiv auth proxy service (10000)"
+  notify: Save iptables rules
diff --git a/roles/contiv/tasks/ovs.yml b/roles/contiv/tasks/ovs.yml
index 5c92e90e9..21ba6ead4 100644
--- a/roles/contiv/tasks/ovs.yml
+++ b/roles/contiv/tasks/ovs.yml
@@ -1,6 +1,6 @@
 ---
 - include_tasks: packageManagerInstall.yml
-  when: source_type == "packageManager"
+  when: contiv_source_type == "packageManager"
   tags:
     - binary-update
 
diff --git a/roles/contiv/tasks/packageManagerInstall.yml b/roles/contiv/tasks/packageManagerInstall.yml
index 3367844a8..8c8e7a7bd 100644
--- a/roles/contiv/tasks/packageManagerInstall.yml
+++ b/roles/contiv/tasks/packageManagerInstall.yml
@@ -4,10 +4,9 @@
     did_install: false
 
 - include_tasks: pkgMgrInstallers/centos-install.yml
-  when: (ansible_os_family == "RedHat") and
-        not openshift_is_atomic
+  when: ansible_os_family == "RedHat" and not openshift_is_atomic | bool
 
 - name: Package Manager | Set fact saying we did CentOS package install
   set_fact:
     did_install: true
-  when: (ansible_os_family == "RedHat")
+  when: ansible_os_family == "RedHat"
diff --git a/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml b/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
index 53c5b4099..2c82973d6 100644
--- a/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
+++ b/roles/contiv/tasks/pkgMgrInstallers/centos-install.yml
@@ -12,9 +12,9 @@
     dest: /tmp/rdo-release-ocata-2.noarch.rpm
     validate_certs: False
   environment:
-    http_proxy: "{{ http_proxy|default('') }}"
-    https_proxy: "{{ https_proxy|default('') }}"
-    no_proxy: "{{ no_proxy|default('') }}"
+    http_proxy: "{{ contiv_http_proxy|default('') }}"
+    https_proxy: "{{ contiv_https_proxy|default('') }}"
+    no_proxy: "{{ contiv_no_proxy|default('') }}"
   tags:
     - ovs_install
 
@@ -30,9 +30,9 @@
     pkg=openvswitch
     state=present
   environment:
-    http_proxy: "{{ http_proxy|default('') }}"
-    https_proxy: "{{ https_proxy|default('') }}"
-    no_proxy: "{{ no_proxy|default('') }}"
+    http_proxy: "{{ contiv_http_proxy|default('') }}"
+    https_proxy: "{{ contiv_https_proxy|default('') }}"
+    no_proxy: "{{ contiv_no_proxy|default('') }}"
   tags:
     - ovs_install
   register: result