From 949f4eacd2aa47833e1283de284aaed1c7ce91fe Mon Sep 17 00:00:00 2001
From: Clayton Coleman <ccoleman@redhat.com>
Date: Thu, 21 Dec 2017 03:14:04 -0500
Subject: Move origin-gce roles and playbooks into openshift-ansible

This moves all core functionality into the openshift-ansible repo, adds
the necessary equivalent entrypoint to the openshift-ansible installer
image, and ensures the dynamic inventory mechanisms in openshift-ansible
continue to work.

Notable changes from origin-gce:

* playbook extensions changed to .yml
* dynamic inventory subdirectory created to prevent accidental use
* use the custom entrypoint entrypoint-gcp for this image
* move tasks into openshift_gcp role
---
 .../openshift-bootstrap-controller-policy.yaml     | 10 ++++
 .../files/openshift-bootstrap-controller-role.yaml | 21 +++++++
 ...nshift-bootstrap-controller-serviceaccount.yaml |  5 ++
 .../files/openshift-bootstrap-controller.yaml      | 68 ++++++++++++++++++++++
 .../tasks/main.yml                                 | 28 +++++++++
 roles/openshift_cloud_provider/tasks/gce.yml       | 10 +++-
 roles/openshift_gcp/files/bootstrap-script.sh      | 42 +++++++++++++
 .../files/openshift-bootstrap-update.service       |  7 +++
 .../files/openshift-bootstrap-update.timer         | 10 ++++
 roles/openshift_gcp/files/partition.conf           |  3 +
 roles/openshift_gcp/meta/main.yml                  | 17 ++++++
 .../tasks/add_custom_repositories.yml              | 20 +++++++
 .../tasks/configure_gcp_base_image.yml             | 10 ++++
 .../tasks/configure_master_bootstrap.yml           | 36 ++++++++++++
 .../tasks/configure_master_healthcheck.yml         | 19 ++++++
 roles/openshift_gcp/tasks/dynamic_inventory.yml    |  5 ++
 .../openshift_gcp/tasks/frequent_log_rotation.yml  | 18 ++++++
 roles/openshift_gcp/tasks/main.yaml                | 43 --------------
 roles/openshift_gcp/tasks/main.yml                 | 43 ++++++++++++++
 roles/openshift_gcp/tasks/node_cloud_config.yml    | 12 ++++
 roles/openshift_gcp/tasks/publish_image.yml        | 32 ++++++++++
 .../tasks/setup_scale_group_facts.yml              | 44 ++++++++++++++
 roles/openshift_gcp/templates/inventory.j2.sh      |  8 +++
 .../openshift_gcp/templates/master_healthcheck.j2  | 68 ++++++++++++++++++++++
 .../templates/openshift-bootstrap-update.j2        |  7 +++
 roles/openshift_gcp/templates/provision.j2.sh      | 17 +++++-
 roles/openshift_gcp/templates/yum_repo.j2          | 20 +++++++
 .../openshift_gcp_image_prep/files/partition.conf  |  3 -
 roles/openshift_gcp_image_prep/tasks/main.yaml     | 18 ------
 29 files changed, 575 insertions(+), 69 deletions(-)
 create mode 100644 roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-policy.yaml
 create mode 100644 roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-role.yaml
 create mode 100644 roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-serviceaccount.yaml
 create mode 100644 roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller.yaml
 create mode 100644 roles/openshift_bootstrap_autoapprover/tasks/main.yml
 create mode 100644 roles/openshift_gcp/files/bootstrap-script.sh
 create mode 100644 roles/openshift_gcp/files/openshift-bootstrap-update.service
 create mode 100644 roles/openshift_gcp/files/openshift-bootstrap-update.timer
 create mode 100644 roles/openshift_gcp/files/partition.conf
 create mode 100644 roles/openshift_gcp/meta/main.yml
 create mode 100644 roles/openshift_gcp/tasks/add_custom_repositories.yml
 create mode 100644 roles/openshift_gcp/tasks/configure_gcp_base_image.yml
 create mode 100644 roles/openshift_gcp/tasks/configure_master_bootstrap.yml
 create mode 100644 roles/openshift_gcp/tasks/configure_master_healthcheck.yml
 create mode 100644 roles/openshift_gcp/tasks/dynamic_inventory.yml
 create mode 100644 roles/openshift_gcp/tasks/frequent_log_rotation.yml
 delete mode 100644 roles/openshift_gcp/tasks/main.yaml
 create mode 100644 roles/openshift_gcp/tasks/main.yml
 create mode 100644 roles/openshift_gcp/tasks/node_cloud_config.yml
 create mode 100644 roles/openshift_gcp/tasks/publish_image.yml
 create mode 100644 roles/openshift_gcp/tasks/setup_scale_group_facts.yml
 create mode 100644 roles/openshift_gcp/templates/inventory.j2.sh
 create mode 100644 roles/openshift_gcp/templates/master_healthcheck.j2
 create mode 100644 roles/openshift_gcp/templates/openshift-bootstrap-update.j2
 create mode 100644 roles/openshift_gcp/templates/yum_repo.j2
 delete mode 100644 roles/openshift_gcp_image_prep/files/partition.conf
 delete mode 100644 roles/openshift_gcp_image_prep/tasks/main.yaml

(limited to 'roles')

diff --git a/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-policy.yaml b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-policy.yaml
new file mode 100644
index 000000000..90ee40943
--- /dev/null
+++ b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-policy.yaml
@@ -0,0 +1,10 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: bootstrap-autoapprover
+roleRef:
+  kind: ClusterRole
+  name: system:node-bootstrap-autoapprover
+subjects:
+- kind: User
+  name: system:serviceaccount:openshift-infra:bootstrap-autoapprover
diff --git a/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-role.yaml b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-role.yaml
new file mode 100644
index 000000000..d8143d047
--- /dev/null
+++ b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-role.yaml
@@ -0,0 +1,21 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: system:node-bootstrap-autoapprover
+rules:
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - create
+  - update
diff --git a/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-serviceaccount.yaml b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-serviceaccount.yaml
new file mode 100644
index 000000000..e22ce6f34
--- /dev/null
+++ b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller-serviceaccount.yaml
@@ -0,0 +1,5 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: bootstrap-autoapprover
+  namespace: openshift-infra
diff --git a/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller.yaml b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller.yaml
new file mode 100644
index 000000000..dbcedb407
--- /dev/null
+++ b/roles/openshift_bootstrap_autoapprover/files/openshift-bootstrap-controller.yaml
@@ -0,0 +1,68 @@
+kind: StatefulSet
+apiVersion: apps/v1beta1
+metadata:
+  name: bootstrap-autoapprover
+  namespace: openshift-infra
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: bootstrap-autoapprover
+    spec:
+      serviceAccountName: bootstrap-autoapprover
+      terminationGracePeriodSeconds: 1
+      containers:
+      - name: signer
+        image: openshift/node:v3.7.0-rc.0
+        command:
+        - /bin/bash
+        - -c
+        args:
+        - |
+          #!/bin/bash
+          set -o errexit
+          set -o nounset
+          set -o pipefail
+
+          unset KUBECONFIG
+          cat <<SCRIPT > /tmp/signer
+          #!/bin/bash
+          #
+          # It will approve any CSR that is not approved yet, and delete any CSR that expired more than 60 seconds
+          # ago.
+          #
+
+          set -o errexit
+          set -o nounset
+          set -o pipefail
+
+          name=\${1}
+          condition=\${2}
+          certificate=\${3}
+          username=\${4}
+
+          # auto approve
+          if [[ -z "\${condition}" && ("\${username}" == "system:serviceaccount:openshift-infra:node-bootstrapper" || "\${username}" == "system:node:"* ) ]]; then
+            oc adm certificate approve "\${name}"
+            exit 0
+          fi
+
+          # check certificate age
+          if [[ -n "\${certificate}" ]]; then
+            text="\$( echo "\${certificate}" | base64 -d - )"
+            if ! echo "\${text}" | openssl x509 -noout; then
+              echo "error: Unable to parse certificate" 2>&1
+              exit 1
+            fi 
+            if ! echo "\${text}" | openssl x509 -checkend -60 > /dev/null; then
+              echo "Certificate is expired, deleting"
+              oc delete csr "\${name}"
+            fi
+            exit 0
+          fi
+          SCRIPT
+          chmod u+x /tmp/signer
+
+          exec oc observe csr --maximum-errors=1 --resync-period=10m -a '{.status.conditions[*].type}' -a '{.status.certificate}' -a '{.spec.username}' -- /tmp/signer
diff --git a/roles/openshift_bootstrap_autoapprover/tasks/main.yml b/roles/openshift_bootstrap_autoapprover/tasks/main.yml
new file mode 100644
index 000000000..88e9d08e7
--- /dev/null
+++ b/roles/openshift_bootstrap_autoapprover/tasks/main.yml
@@ -0,0 +1,28 @@
+---
+- name: Copy auto-approver config to host
+  run_once: true
+  copy:
+    src: "{{ item }}"
+    dest: /tmp/openshift-approver/
+    owner: root
+    mode: 0400
+  with_fileglob:
+    - "*.yaml"
+
+- name: Set auto-approver nodeSelector
+  run_once: true
+  yedit:
+    src: "/tmp/openshift-approver/openshift-bootstrap-controller.yaml"
+    key: spec.template.spec.nodeSelector
+    value: "{{ openshift_master_bootstrap_auto_approver_node_selector | default({}) }}"
+    value_type: list
+
+- name: Create auto-approver on cluster
+  run_once: true
+  command: oc apply -f /tmp/openshift-approver/
+
+- name: Remove auto-approver config
+  run_once: true
+  file:
+    path: /tmp/openshift-approver/
+    state: absent
diff --git a/roles/openshift_cloud_provider/tasks/gce.yml b/roles/openshift_cloud_provider/tasks/gce.yml
index ee4048911..395bd304c 100644
--- a/roles/openshift_cloud_provider/tasks/gce.yml
+++ b/roles/openshift_cloud_provider/tasks/gce.yml
@@ -13,5 +13,11 @@
   ini_file:
     dest: "{{ openshift.common.config_base }}/cloudprovider/gce.conf"
     section: Global
-    option: multizone
-    value: "true"
+    option: "{{ item.key }}"
+    value: "{{ item.value }}"
+  with_items:
+    - { key: 'project-id', value: '{{ openshift_gcp_project }}' }
+    - { key: 'network-name', value: '{{ openshift_gcp_network_name }}' }
+    - { key: 'node-tags', value: '{{ openshift_gcp_prefix }}ocp' }
+    - { key: 'node-instance-prefix', value: '{{ openshift_gcp_prefix }}' }
+    - { key: 'multizone', value: 'false' }
diff --git a/roles/openshift_gcp/files/bootstrap-script.sh b/roles/openshift_gcp/files/bootstrap-script.sh
new file mode 100644
index 000000000..0c3f1999b
--- /dev/null
+++ b/roles/openshift_gcp/files/bootstrap-script.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# This script is a startup script for bootstrapping a GCP node
+# from a config stored in the project metadata. It loops until
+# it finds the script and then starts the origin-node service.
+# TODO: generalize
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap" -H "Metadata-Flavor: Google" )" != "true" ]]; then
+  echo "info: Bootstrap is not enabled for this instance, skipping" 1>&2
+  exit 0
+fi
+
+if ! id=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-id" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get cluster-id for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! node_group=$( curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/node-group" -H "Metadata-Flavor: Google" ); then
+  echo "error: Unable to get node-group for instance from cluster metadata" 1>&2
+  exit 1
+fi
+
+if ! config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+  while true; do
+    if config=$( curl -f "http://metadata.google.internal/computeMetadata/v1/project/attributes/${id}-bootstrap-config" -H "Metadata-Flavor: Google" 2>/dev/null ); then
+      break
+    fi
+    echo "info: waiting for ${id}-bootstrap-config to become available in cluster metadata ..." 1>&2
+    sleep 5
+  done
+fi
+
+echo "Got bootstrap config from metadata"
+mkdir -p /etc/origin/node
+echo -n "${config}" > /etc/origin/node/bootstrap.kubeconfig
+echo "BOOTSTRAP_CONFIG_NAME=node-config-${node_group}" >> /etc/sysconfig/origin-node
+systemctl enable origin-node
+systemctl start origin-node
diff --git a/roles/openshift_gcp/files/openshift-bootstrap-update.service b/roles/openshift_gcp/files/openshift-bootstrap-update.service
new file mode 100644
index 000000000..c65b1b34e
--- /dev/null
+++ b/roles/openshift_gcp/files/openshift-bootstrap-update.service
@@ -0,0 +1,7 @@
+[Unit]
+Description=Update the OpenShift node bootstrap configuration
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/openshift-bootstrap-update
+User=root
diff --git a/roles/openshift_gcp/files/openshift-bootstrap-update.timer b/roles/openshift_gcp/files/openshift-bootstrap-update.timer
new file mode 100644
index 000000000..1a517b33e
--- /dev/null
+++ b/roles/openshift_gcp/files/openshift-bootstrap-update.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=Update the OpenShift node bootstrap credentials hourly
+
+[Timer]
+OnBootSec=30s
+OnCalendar=hourly
+Persistent=true     
+ 
+[Install]
+WantedBy=timers.target
\ No newline at end of file
diff --git a/roles/openshift_gcp/files/partition.conf b/roles/openshift_gcp/files/partition.conf
new file mode 100644
index 000000000..76e65ab9c
--- /dev/null
+++ b/roles/openshift_gcp/files/partition.conf
@@ -0,0 +1,3 @@
+[Service]
+ExecStartPost=-/usr/bin/growpart /dev/sda 1
+ExecStartPost=-/sbin/xfs_growfs /
\ No newline at end of file
diff --git a/roles/openshift_gcp/meta/main.yml b/roles/openshift_gcp/meta/main.yml
new file mode 100644
index 000000000..5e428f8de
--- /dev/null
+++ b/roles/openshift_gcp/meta/main.yml
@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  author: Clayton Coleman
+  description:
+  company: Red Hat, Inc.
+  license: Apache License, Version 2.0
+  min_ansible_version: 1.8
+  platforms:
+  - name: EL
+    versions:
+    - 7
+  categories:
+  - cloud
+  - system
+dependencies:
+- role: lib_utils
+- role: lib_openshift
diff --git a/roles/openshift_gcp/tasks/add_custom_repositories.yml b/roles/openshift_gcp/tasks/add_custom_repositories.yml
new file mode 100644
index 000000000..04718f78e
--- /dev/null
+++ b/roles/openshift_gcp/tasks/add_custom_repositories.yml
@@ -0,0 +1,20 @@
+---
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientcert }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_cert
+  when: item.1.sslclientcert | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+- name: Copy custom repository secrets
+  copy:
+    src: "{{ files_dir }}/{{ item.1.sslclientkey }}"
+    dest: /var/lib/yum/custom_secret_{{ item.0 }}_key
+  when: item.1.sslclientkey | default(false)
+  with_indexed_items: "{{ provision_custom_repositories }}"
+
+- name: Create any custom repos that are defined
+  template:
+    src: yum_repo.j2
+    dest: /etc/yum.repos.d/provision_custom_repositories.repo
+  when: provision_custom_repositories | length > 0
+  notify: refresh cache
diff --git a/roles/openshift_gcp/tasks/configure_gcp_base_image.yml b/roles/openshift_gcp/tasks/configure_gcp_base_image.yml
new file mode 100644
index 000000000..2c6e2790a
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_gcp_base_image.yml
@@ -0,0 +1,10 @@
+# GCE instances are starting with xfs AND barrier=1, which is only for extfs.
+---
+- name: Remove barrier=1 from XFS fstab entries
+  command: sed -i -e 's/xfs\(.*\)barrier=1/xfs\1/g; s/, / /g' /etc/fstab
+
+- name: Ensure the root filesystem has XFS group quota turned on
+  command: sed -i -e 's/linux16 \(.*\)$/linux16 \1 rootflags=gquota/g' /boot/grub2/grub.cfg
+
+- name: Ensure the root partition grows on startup
+  copy: src=partition.conf dest=/etc/systemd/system/google-instance-setup.service.d/
diff --git a/roles/openshift_gcp/tasks/configure_master_bootstrap.yml b/roles/openshift_gcp/tasks/configure_master_bootstrap.yml
new file mode 100644
index 000000000..591cb593c
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_master_bootstrap.yml
@@ -0,0 +1,36 @@
+#
+# These tasks configure the instance to periodically update the project metadata with the
+# latest bootstrap kubeconfig from the project metadata. This keeps the project metadata
+# in sync with the cluster's configuration. We then invoke a CSR approve on any nodes that
+# are waiting to join the cluster.
+#
+---
+- name: Copy unit service
+  copy:
+    src: openshift-bootstrap-update.timer
+    dest: /etc/systemd/system/openshift-bootstrap-update.timer
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Copy unit timer
+  copy:
+    src: openshift-bootstrap-update.service
+    dest: /etc/systemd/system/openshift-bootstrap-update.service
+    owner: root
+    group: root
+    mode: 0664
+
+- name: Create bootstrap update script
+  template: src=openshift-bootstrap-update.j2 dest=/usr/bin/openshift-bootstrap-update mode=u+rx
+
+- name: Start bootstrap update timer
+  systemd:
+    name: "openshift-bootstrap-update.timer"
+    state: started
+
+- name: Bootstrap all nodes that were identified with bootstrap metadata
+  run_once: true
+  oc_adm_csr:
+    nodes: "{{ groups['all'] | map('extract', hostvars) | selectattr('gce_metadata.bootstrap', 'match', 'true') | map(attribute='gce_name') | list }}"
+    timeout: 60
diff --git a/roles/openshift_gcp/tasks/configure_master_healthcheck.yml b/roles/openshift_gcp/tasks/configure_master_healthcheck.yml
new file mode 100644
index 000000000..aa9655977
--- /dev/null
+++ b/roles/openshift_gcp/tasks/configure_master_healthcheck.yml
@@ -0,0 +1,19 @@
+---
+- name: refresh yum cache
+  command: yum clean all
+  args:
+    warn: no
+  when: ansible_os_family == "RedHat"
+
+- name: install haproxy
+  package: name=haproxy state=present
+  register: result
+  until: '"failed" not in result'
+  retries: 10
+  delay: 10
+
+- name: configure haproxy
+  template: src=master_healthcheck.j2 dest=/etc/haproxy/haproxy.cfg
+
+- name: start and enable haproxy service
+  service: name=haproxy state=started enabled=yes
diff --git a/roles/openshift_gcp/tasks/dynamic_inventory.yml b/roles/openshift_gcp/tasks/dynamic_inventory.yml
new file mode 100644
index 000000000..1637da945
--- /dev/null
+++ b/roles/openshift_gcp/tasks/dynamic_inventory.yml
@@ -0,0 +1,5 @@
+---
+- name: Extract PEM from service account file
+  copy: content="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).private_key }}" dest=/tmp/gce.pem mode=0600
+- name: Templatize environment script
+  template: src=inventory.j2.sh dest=/tmp/inventory.sh mode=u+rx
diff --git a/roles/openshift_gcp/tasks/frequent_log_rotation.yml b/roles/openshift_gcp/tasks/frequent_log_rotation.yml
new file mode 100644
index 000000000..0b4b27f84
--- /dev/null
+++ b/roles/openshift_gcp/tasks/frequent_log_rotation.yml
@@ -0,0 +1,18 @@
+---
+- name: Rotate logs daily
+  replace:
+    dest: /etc/logrotate.conf
+    regexp: '^weekly|monthly|yearly$'
+    replace: daily
+- name: Rotate at a smaller size of log
+  lineinfile:
+    dest: /etc/logrotate.conf
+    state: present
+    regexp: '^size'
+    line: size 10M
+- name: Limit total size of log files
+  lineinfile:
+    dest: /etc/logrotate.conf
+    state: present
+    regexp: '^maxsize'
+    line: maxsize 20M
diff --git a/roles/openshift_gcp/tasks/main.yaml b/roles/openshift_gcp/tasks/main.yaml
deleted file mode 100644
index ad205ba33..000000000
--- a/roles/openshift_gcp/tasks/main.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# This role relies on gcloud invoked via templated bash in order to
-# provide a high performance deployment option. The next logical step
-# is to transition to a deployment manager template which is then instantiated.
-# TODO: use a formal set of role parameters consistent with openshift_aws
-#
----
-- name: Templatize DNS script
-  template: src=dns.j2.sh dest=/tmp/openshift_gcp_provision_dns.sh mode=u+rx
-- name: Templatize provision script
-  template: src=provision.j2.sh dest=/tmp/openshift_gcp_provision.sh mode=u+rx
-- name: Templatize de-provision script
-  template: src=remove.j2.sh dest=/tmp/openshift_gcp_provision_remove.sh mode=u+rx
-  when:
-  - state | default('present') == 'absent'
-
-- name: Provision GCP DNS domain
-  command: /tmp/openshift_gcp_provision_dns.sh
-  args:
-    chdir: "{{ playbook_dir }}/files"
-  register: dns_provision
-  when:
-  - state | default('present') == 'present'
-
-- name: Ensure that DNS resolves to the hosted zone
-  assert:
-    that:
-    - "lookup('dig', public_hosted_zone, 'qtype=NS', wantlist=True) | sort | join(',') == dns_provision.stdout"
-    msg: "The DNS domain {{ public_hosted_zone }} defined in 'public_hosted_zone' must have NS records pointing to the Google nameservers: '{{ dns_provision.stdout }}' instead of '{{ lookup('dig', public_hosted_zone, 'qtype=NS') }}'."
-  when:
-  - state | default('present') == 'present'
-
-- name: Provision GCP resources
-  command: /tmp/openshift_gcp_provision.sh
-  args:
-    chdir: "{{ playbook_dir }}/files"
-  when:
-  - state | default('present') == 'present'
-
-- name: De-provision GCP resources
-  command: /tmp/openshift_gcp_provision_remove.sh
-  when:
-  - state | default('present') == 'absent'
diff --git a/roles/openshift_gcp/tasks/main.yml b/roles/openshift_gcp/tasks/main.yml
new file mode 100644
index 000000000..fb147bc78
--- /dev/null
+++ b/roles/openshift_gcp/tasks/main.yml
@@ -0,0 +1,43 @@
+#
+# This role relies on gcloud invoked via templated bash in order to
+# provide a high performance deployment option. The next logical step
+# is to transition to a deployment manager template which is then instantiated.
+# TODO: use a formal set of role parameters consistent with openshift_aws
+#
+---
+- name: Templatize DNS script
+  template: src=dns.j2.sh dest=/tmp/openshift_gcp_provision_dns.sh mode=u+rx
+- name: Templatize provision script
+  template: src=provision.j2.sh dest=/tmp/openshift_gcp_provision.sh mode=u+rx
+- name: Templatize de-provision script
+  template: src=remove.j2.sh dest=/tmp/openshift_gcp_provision_remove.sh mode=u+rx
+  when:
+  - state | default('present') == 'absent'
+
+- name: Provision GCP DNS domain
+  command: /tmp/openshift_gcp_provision_dns.sh
+  args:
+    chdir: "{{ files_dir }}"
+  register: dns_provision
+  when:
+  - state | default('present') == 'present'
+
+- name: Ensure that DNS resolves to the hosted zone
+  assert:
+    that:
+    - "lookup('dig', public_hosted_zone, 'qtype=NS', wantlist=True) | sort | join(',') == dns_provision.stdout"
+    msg: "The DNS domain {{ public_hosted_zone }} defined in 'public_hosted_zone' must have NS records pointing to the Google nameservers: '{{ dns_provision.stdout }}' instead of '{{ lookup('dig', public_hosted_zone, 'qtype=NS') }}'."
+  when:
+  - state | default('present') == 'present'
+
+- name: Provision GCP resources
+  command: /tmp/openshift_gcp_provision.sh
+  args:
+    chdir: "{{ files_dir }}"
+  when:
+  - state | default('present') == 'present'
+
+- name: De-provision GCP resources
+  command: /tmp/openshift_gcp_provision_remove.sh
+  when:
+  - state | default('present') == 'absent'
diff --git a/roles/openshift_gcp/tasks/node_cloud_config.yml b/roles/openshift_gcp/tasks/node_cloud_config.yml
new file mode 100644
index 000000000..4e982f497
--- /dev/null
+++ b/roles/openshift_gcp/tasks/node_cloud_config.yml
@@ -0,0 +1,12 @@
+---
+- name: ensure the /etc/origin folder exists
+  file: name=/etc/origin state=directory
+
+- name: configure gce cloud config options
+  ini_file: dest=/etc/origin/cloudprovider/gce.conf section=Global option={{ item.key }} value={{ item.value }} state=present create=yes
+  with_items:
+    - { key: 'project-id', value: '{{ openshift_gcp_project }}' }
+    - { key: 'network-name', value: '{{ openshift_gcp_network_name }}' }
+    - { key: 'node-tags', value: '{{ openshift_gcp_prefix }}ocp' }
+    - { key: 'node-instance-prefix', value: '{{ openshift_gcp_prefix }}' }
+    - { key: 'multizone', value: 'false' }
diff --git a/roles/openshift_gcp/tasks/publish_image.yml b/roles/openshift_gcp/tasks/publish_image.yml
new file mode 100644
index 000000000..db8a7ca69
--- /dev/null
+++ b/roles/openshift_gcp/tasks/publish_image.yml
@@ -0,0 +1,32 @@
+---
+- name: Require openshift_gcp_image
+  fail:
+    msg: "A source image name or family is required for image publishing.  Please ensure `openshift_gcp_image` is defined."
+  when: openshift_gcp_image is undefined
+
+- name: Require openshift_gcp_target_image
+  fail:
+    msg: "A target image name or family is required for image publishing.  Please ensure `openshift_gcp_target_image` is defined."
+  when: openshift_gcp_target_image is undefined
+
+- block:
+  - name: Retrieve images in the {{ openshift_gcp_target_image }} family
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images list
+        "--filter=family={{ openshift_gcp_target_image }}"
+        --format=json --sort-by ~creationTimestamp
+    register: images
+  - name: Prune oldest images
+    command: >
+      gcloud --project "{{ openshift_gcp_project }}" compute images delete "{{ item['name'] }}"
+    with_items: "{{ (images.stdout | default('[]') | from_json )[( openshift_gcp_keep_images | int ):] }}"
+  when: openshift_gcp_keep_images is defined
+
+- name: Copy the latest image in the family {{ openshift_gcp_image }} to {{ openshift_gcp_target_image }}
+  command: >
+    gcloud --project "{{ openshift_gcp_target_project | default(openshift_gcp_project) }}"
+      beta compute images create
+      "{{ openshift_gcp_target_image_name | default(openshift_gcp_target_image + '-' + lookup('pipe','date +%Y%m%d-%H%M%S')) }}"
+      --family "{{ openshift_gcp_target_image }}"
+      --source-image-family "{{ openshift_gcp_image }}"
+      --source-image-project "{{ openshift_gcp_project }}"
diff --git a/roles/openshift_gcp/tasks/setup_scale_group_facts.yml b/roles/openshift_gcp/tasks/setup_scale_group_facts.yml
new file mode 100644
index 000000000..0fda43123
--- /dev/null
+++ b/roles/openshift_gcp/tasks/setup_scale_group_facts.yml
@@ -0,0 +1,44 @@
+---
+- name: Add masters to requisite groups
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: masters, etcd
+  with_items: "{{ groups['tag_ocp-master'] }}"
+
+- name: Add a master to the primary masters group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: primary_master
+  with_items: "{{ groups['tag_ocp-master'].0 }}"
+
+- name: Add non-bootstrapping master node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: infra
+  with_items: "{{ groups['tag_ocp-master'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add infra node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: infra
+  with_items: "{{ groups['tag_ocp-infra-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add node instances to node group
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: nodes
+    openshift_node_labels:
+      role: app
+  with_items: "{{ groups['tag_ocp-node'] | default([]) | difference(groups['tag_ocp-bootstrap'] | default([])) }}"
+
+- name: Add bootstrap node instances
+  add_host:
+    name: "{{ hostvars[item].gce_name }}"
+    groups: bootstrap_nodes
+    openshift_node_bootstrap: True
+  with_items: "{{ groups['tag_ocp-node'] | default([]) | intersect(groups['tag_ocp-bootstrap'] | default([])) }}"
+  when: not (openshift_node_bootstrap | default(False))
diff --git a/roles/openshift_gcp/templates/inventory.j2.sh b/roles/openshift_gcp/templates/inventory.j2.sh
new file mode 100644
index 000000000..dcaffb578
--- /dev/null
+++ b/roles/openshift_gcp/templates/inventory.j2.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+export GCE_PROJECT="{{ openshift_gcp_project }}"
+export GCE_ZONE="{{ openshift_gcp_zone }}"
+export GCE_EMAIL="{{ (lookup('file', openshift_gcp_iam_service_account_keyfile ) | from_json ).client_email }}"
+export GCE_PEM_FILE_PATH="/tmp/gce.pem"
+export INVENTORY_IP_TYPE="{{ inventory_ip_type }}"
+export GCE_TAGGED_INSTANCES="{{ openshift_gcp_prefix }}ocp"
\ No newline at end of file
diff --git a/roles/openshift_gcp/templates/master_healthcheck.j2 b/roles/openshift_gcp/templates/master_healthcheck.j2
new file mode 100644
index 000000000..189e578c5
--- /dev/null
+++ b/roles/openshift_gcp/templates/master_healthcheck.j2
@@ -0,0 +1,68 @@
+#---------------------------------------------------------------------
+# Example configuration for a possible web application.  See the
+# full configuration options online.
+#
+#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
+#
+#---------------------------------------------------------------------
+
+#---------------------------------------------------------------------
+# Global settings
+#---------------------------------------------------------------------
+global
+    # to have these messages end up in /var/log/haproxy.log you will
+    # need to:
+    #
+    # 1) configure syslog to accept network log events.  This is done
+    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
+    #    /etc/sysconfig/syslog
+    #
+    # 2) configure local2 events to go to the /var/log/haproxy.log
+    #   file. A line like the following can be added to
+    #   /etc/sysconfig/syslog
+    #
+    #    local2.*                       /var/log/haproxy.log
+    #
+    log         127.0.0.1 local2
+
+    chroot      /var/lib/haproxy
+    pidfile     /var/run/haproxy.pid
+    maxconn     4000
+    user        haproxy
+    group       haproxy
+    daemon
+
+    # turn on stats unix socket
+    stats socket /var/lib/haproxy/stats
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 3
+    timeout http-request    10s
+    timeout queue           1m
+    timeout connect         10s
+    timeout client          1m
+    timeout server          1m
+    timeout http-keep-alive 10s
+    timeout check           10s
+    maxconn                 3000
+
+#---------------------------------------------------------------------
+# main frontend which proxys to the backends
+#---------------------------------------------------------------------
+frontend  http-proxy *:8080
+    acl          url_healthz  path_beg  -i /healthz
+    use_backend  ocp          if url_healthz
+
+backend ocp
+    server       ocp localhost:{{ internal_console_port }} ssl verify none
diff --git a/roles/openshift_gcp/templates/openshift-bootstrap-update.j2 b/roles/openshift_gcp/templates/openshift-bootstrap-update.j2
new file mode 100644
index 000000000..5b0563724
--- /dev/null
+++ b/roles/openshift_gcp/templates/openshift-bootstrap-update.j2
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+oc serviceaccounts create-kubeconfig -n openshift-infra node-bootstrapper > /root/bootstrap.kubeconfig
+gcloud compute project-info --project '{{ openshift_gcp_project }}' add-metadata --metadata-from-file '{{ openshift_gcp_prefix + openshift_gcp_clusterid | default("default") }}-bootstrap-config=/root/bootstrap.kubeconfig'
+rm -f /root/bootstrap.kubeconfig
diff --git a/roles/openshift_gcp/templates/provision.j2.sh b/roles/openshift_gcp/templates/provision.j2.sh
index 4d150bc74..794985322 100644
--- a/roles/openshift_gcp/templates/provision.j2.sh
+++ b/roles/openshift_gcp/templates/provision.j2.sh
@@ -9,15 +9,26 @@ if [[ -n "{{ openshift_gcp_ssh_private_key }}" ]]; then
         ssh-add "{{ openshift_gcp_ssh_private_key }}" || true
     fi
 
-    # Check if the ~/.ssh/google_compute_engine.pub key is in the project metadata, and if not, add it there
-    pub_key=$(cut -d ' ' -f 2 < "{{ openshift_gcp_ssh_private_key }}.pub")
+    # Check if the public key is in the project metadata, and if not, add it there
+    if [ -f "{{ openshift_gcp_ssh_private_key }}.pub" ]; then
+        pub_file="{{ openshift_gcp_ssh_private_key }}.pub"
+        pub_key=$(cut -d ' ' -f 2 < "{{ openshift_gcp_ssh_private_key }}.pub")
+    else
+        keyfile="${HOME}/.ssh/google_compute_engine"
+        pub_file="${keyfile}.pub"
+        mkdir -p "${HOME}/.ssh"
+        cp "{{ openshift_gcp_ssh_private_key }}" "${keyfile}"
+        chmod 0600 "${keyfile}"
+        ssh-keygen -y -f "${keyfile}" >  "${pub_file}"
+        pub_key=$(cut -d ' ' -f 2 <  "${pub_file}")
+    fi
     key_tmp_file='/tmp/ocp-gce-keys'
     if ! gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep -q "$pub_key"; then
         if gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep -q ssh-rsa; then
             gcloud --project "{{ openshift_gcp_project }}" compute project-info describe | grep ssh-rsa | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/value: //' > "$key_tmp_file"
         fi
         echo -n 'cloud-user:' >> "$key_tmp_file"
-        cat "{{ openshift_gcp_ssh_private_key }}.pub" >> "$key_tmp_file"
+        cat "${pub_file}" >> "$key_tmp_file"
         gcloud --project "{{ openshift_gcp_project }}" compute project-info add-metadata --metadata-from-file "sshKeys=${key_tmp_file}"
         rm -f "$key_tmp_file"
     fi
diff --git a/roles/openshift_gcp/templates/yum_repo.j2 b/roles/openshift_gcp/templates/yum_repo.j2
new file mode 100644
index 000000000..77919ea75
--- /dev/null
+++ b/roles/openshift_gcp/templates/yum_repo.j2
@@ -0,0 +1,20 @@
+{% for repo in provision_custom_repositories %}
+[{{ repo.id | default(repo.name) }}]
+name={{ repo.name | default(repo.id) }}
+baseurl={{ repo.baseurl }}
+{% set enable_repo = repo.enabled | default(1) %}
+enabled={{ 1 if ( enable_repo == 1 or enable_repo == True ) else 0 }}
+{% set enable_gpg_check = repo.gpgcheck | default(1) %}
+gpgcheck={{ 1 if ( enable_gpg_check == 1 or enable_gpg_check == True ) else 0 }}
+{% if 'sslclientcert' in repo %}
+sslclientcert={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_cert" if repo.sslclientcert }}
+{% endif %}
+{% if 'sslclientkey' in repo %}
+sslclientkey={{ "/var/lib/yum/custom_secret_" + (loop.index-1)|string + "_key" if repo.sslclientkey }}
+{% endif %}
+{% for key, value in repo.iteritems() %}
+{% if key not in ['id', 'name', 'baseurl', 'enabled', 'gpgcheck', 'sslclientkey', 'sslclientcert'] and value is defined %}
+{{ key }}={{ value }}
+{% endif %}
+{% endfor %}
+{% endfor %}
diff --git a/roles/openshift_gcp_image_prep/files/partition.conf b/roles/openshift_gcp_image_prep/files/partition.conf
deleted file mode 100644
index b87e5e0b6..000000000
--- a/roles/openshift_gcp_image_prep/files/partition.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-[Service]
-ExecStartPost=-/usr/bin/growpart /dev/sda 1
-ExecStartPost=-/sbin/xfs_growfs /
diff --git a/roles/openshift_gcp_image_prep/tasks/main.yaml b/roles/openshift_gcp_image_prep/tasks/main.yaml
deleted file mode 100644
index fee5ab618..000000000
--- a/roles/openshift_gcp_image_prep/tasks/main.yaml
+++ /dev/null
@@ -1,18 +0,0 @@
----
-# GCE instances are starting with xfs AND barrier=1, which is only for extfs.
-- name: Remove barrier=1 from XFS fstab entries
-  lineinfile:
-    path: /etc/fstab
-    regexp: '^(.+)xfs(.+?),?barrier=1,?(.*?)$'
-    line: '\1xfs\2 \4'
-    backrefs: yes
-
-- name: Ensure the root filesystem has XFS group quota turned on
-  lineinfile:
-    path: /boot/grub2/grub.cfg
-    regexp: '^(.*)linux16 (.*)$'
-    line: '\1linux16 \2 rootflags=gquota'
-    backrefs: yes
-
-- name: Ensure the root partition grows on startup
-  copy: src=partition.conf dest=/etc/systemd/system/google-instance-setup.service.d/
-- 
cgit v1.2.1