summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--inventory/byo/hosts.byo.native-glusterfs.example2
-rw-r--r--roles/openshift_storage_glusterfs/README.md83
-rw-r--r--roles/openshift_storage_glusterfs/defaults/main.yml16
-rw-r--r--roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml50
-rw-r--r--roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml31
-rw-r--r--roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml41
-rw-r--r--roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml108
-rw-r--r--roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml6
-rw-r--r--roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml36
-rw-r--r--roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml29
-rw-r--r--roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml21
-rw-r--r--roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml68
-rw-r--r--roles/openshift_storage_glusterfs/tasks/main.yml5
-rw-r--r--roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j23
-rw-r--r--roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 (renamed from roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml)2
-rw-r--r--roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j210
-rw-r--r--roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j212
-rw-r--r--roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j210
18 files changed, 357 insertions, 176 deletions
diff --git a/inventory/byo/hosts.byo.native-glusterfs.example b/inventory/byo/hosts.byo.native-glusterfs.example
index 2dbb57d40..dc847a5b2 100644
--- a/inventory/byo/hosts.byo.native-glusterfs.example
+++ b/inventory/byo/hosts.byo.native-glusterfs.example
@@ -24,7 +24,7 @@ glusterfs
[OSEv3:vars]
ansible_ssh_user=root
-deployment_type=origin
+openshift_deployment_type=origin
# Specify that we want to use GlusterFS storage for a hosted registry
openshift_hosted_registry_storage_kind=glusterfs
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md
index 7b310dbf8..62fc35299 100644
--- a/roles/openshift_storage_glusterfs/README.md
+++ b/roles/openshift_storage_glusterfs/README.md
@@ -1,7 +1,31 @@
OpenShift GlusterFS Cluster
===========================
-OpenShift GlusterFS Cluster Installation
+OpenShift GlusterFS Cluster Configuration
+
+This role handles the configuration of GlusterFS clusters. It can handle
+two primary configuration scenarios:
+
+* Configuring a new, natively-hosted GlusterFS cluster. In this scenario,
+ GlusterFS pods are deployed on nodes in the OpenShift cluster which are
+ configured to provide storage.
+* Configuring a new, external GlusterFS cluster. In this scenario, the
+ cluster nodes have the GlusterFS software pre-installed but have not
+ been configured yet. The installer will take care of configuring the
+ cluster(s) for use by OpenShift applications.
+* Using existing GlusterFS clusters. In this scenario, one or more
+ GlusterFS clusters are assumed to be already setup. These clusters can
+ be either natively-hosted or external, but must be managed by a
+ [heketi service](https://github.com/heketi/heketi).
+
+As part of the configuration, a particular GlusterFS cluster may be
+specified to provide backend storage for a natively-hosted Docker
+registry.
+
+Unless configured otherwise, a StorageClass will be automatically
+created for each non-registry GlusterFS cluster. This will allow
+applications which can mount PersistentVolumes to request
+dynamically-provisioned GlusterFS volumes.
Requirements
------------
@@ -21,26 +45,50 @@ hosted Docker registry:
* `[glusterfs_registry]`
+Host Variables
+--------------
+
+For configuring new clusters, the following role variables are available.
+
+Each host in either of the above groups must have the following variable
+defined:
+
+| Name | Default value | Description |
+|-------------------|---------------|-----------------------------------------|
+| glusterfs_devices | None | A list of block devices that will be completely managed as part of a GlusterFS cluster. There must be at least one device listed. Each device must be bare, e.g. no partitions or LVM PVs. **Example:** '[ "/dev/sdb" ]'
+
+In addition, each host may specify the following variables to further control
+their configuration as GlusterFS nodes:
+
+| Name | Default value | Description |
+|--------------------|---------------------------|-----------------------------------------|
+| glusterfs_cluster | 1 | The ID of the cluster this node should belong to. This is useful when a single heketi service is expected to manage multiple distinct clusters. **NOTE:** For natively-hosted clusters, all pods will be in the same OpenShift namespace
+| glusterfs_hostname | openshift.common.hostname | A hostname (or IP address) that will be used for internal GlusterFS communication
+| glusterfs_ip | openshift.common.ip | An IP address that will be used by pods to communicate with the GlusterFS node
+| glusterfs_zone | 1 | A zone number for the node. Zones are used within the cluster for determining how to distribute the bricks of GlusterFS volumes. heketi will try to spread each volumes' bricks as evenly as possible across all zones
+
Role Variables
--------------
This role has the following variables that control the integration of a
GlusterFS cluster into a new or existing OpenShift cluster:
-| Name | Default value | |
+| Name | Default value | Description |
|--------------------------------------------------|-------------------------|-----------------------------------------|
| openshift_storage_glusterfs_timeout | 300 | Seconds to wait for pods to become ready
| openshift_storage_glusterfs_namespace | 'default' | Namespace in which to create GlusterFS resources
| openshift_storage_glusterfs_is_native | True | GlusterFS should be containerized
-| openshift_storage_glusterfs_nodeselector | 'storagenode=glusterfs' | Selector to determine which nodes will host GlusterFS pods in native mode
+| openshift_storage_glusterfs_name | 'storage' | A name to identify the GlusterFS cluster, which will be used in resource names
+| openshift_storage_glusterfs_nodeselector | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name
+| openshift_storage_glusterfs_storageclass | True | Automatically create a StorageClass for each GlusterFS cluster
| openshift_storage_glusterfs_image | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7'
| openshift_storage_glusterfs_version | 'latest' | Container image version to use for GlusterFS pods
| openshift_storage_glusterfs_wipe | False | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.**
| openshift_storage_glusterfs_heketi_is_native | True | heketi should be containerized
| openshift_storage_glusterfs_heketi_image | 'heketi/heketi' | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7'
| openshift_storage_glusterfs_heketi_version | 'latest' | Container image version to use for heketi pods
-| openshift_storage_glusterfs_heketi_admin_key | '' | String to use as secret key for performing heketi commands as admin
-| openshift_storage_glusterfs_heketi_user_key | '' | String to use as secret key for performing heketi commands as user that can only view or modify volumes
+| openshift_storage_glusterfs_heketi_admin_key | auto-generated | String to use as secret key for performing heketi commands as admin
+| openshift_storage_glusterfs_heketi_user_key | auto-generated | String to use as secret key for performing heketi commands as user that can only view or modify volumes
| openshift_storage_glusterfs_heketi_topology_load | True | Load the GlusterFS topology information into heketi
| openshift_storage_glusterfs_heketi_url | Undefined | URL for the heketi REST API, dynamically determined in native mode
| openshift_storage_glusterfs_heketi_wipe | False | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe`
@@ -52,17 +100,24 @@ registry. These variables start with the prefix
values in their corresponding non-registry variables. The following variables
are an exception:
-| Name | Default value | |
-|---------------------------------------------------|-----------------------|-----------------------------------------|
-| openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default'
-| openshift_storage_glusterfs_registry_nodeselector | 'storagenode=registry'| This allows for the logical separation of the registry GlusterFS cluster from any regular-use GlusterFS clusters
+| Name | Default value | Description |
+|-------------------------------------------------------|-----------------------|-----------------------------------------|
+| openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default'
+| openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters
+| openshift_storage_glusterfs_registry_storageclass | False | It is recommended to not create a StorageClass for GlusterFS clusters serving registry storage, so as to avoid performance penalties
+| openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated | Separate from the above
+| openshift_storage_glusterfs_registry_heketi_user_key | auto-generated | Separate from the above
Additionally, this role's behavior responds to the following registry-specific
-variable:
-
-| Name | Default value | Description |
-|----------------------------------------------|---------------|------------------------------------------------------------------------------|
-| openshift_hosted_registry_glusterfs_swap | False | Whether to swap an existing registry's storage volume for a GlusterFS volume |
+variables:
+
+| Name | Default value | Description |
+|-----------------------------------------------|------------------------------|-----------------------------------------|
+| openshift_hosted_registry_glusterfs_endpoints | glusterfs-registry-endpoints | The name for the Endpoints resource that will point the registry to the GlusterFS nodes
+| openshift_hosted_registry_glusterfs_path | glusterfs-registry-volume | The name for the GlusterFS volume that will provide registry storage
+| openshift_hosted_registry_glusterfs_readonly | False | Whether the GlusterFS volume should be read-only
+| openshift_hosted_registry_glusterfs_swap | False | Whether to swap an existing registry's storage volume for a GlusterFS volume
+| openshift_hosted_registry_glusterfs_swapcopy | True | If swapping, copy the contents of the pre-existing registry storage to the new GlusterFS volume
Dependencies
------------
diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml
index ebe9ca30b..468877e57 100644
--- a/roles/openshift_storage_glusterfs/defaults/main.yml
+++ b/roles/openshift_storage_glusterfs/defaults/main.yml
@@ -2,7 +2,9 @@
openshift_storage_glusterfs_timeout: 300
openshift_storage_glusterfs_namespace: 'default'
openshift_storage_glusterfs_is_native: True
-openshift_storage_glusterfs_nodeselector: 'storagenode=glusterfs'
+openshift_storage_glusterfs_name: 'storage'
+openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host"
+openshift_storage_glusterfs_storageclass: True
openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}"
openshift_storage_glusterfs_version: 'latest'
openshift_storage_glusterfs_wipe: False
@@ -11,8 +13,8 @@ openshift_storage_glusterfs_heketi_is_missing: True
openshift_storage_glusterfs_heketi_deploy_is_missing: True
openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}"
openshift_storage_glusterfs_heketi_version: 'latest'
-openshift_storage_glusterfs_heketi_admin_key: ''
-openshift_storage_glusterfs_heketi_user_key: ''
+openshift_storage_glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}"
openshift_storage_glusterfs_heketi_topology_load: True
openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}"
openshift_storage_glusterfs_heketi_url: "{{ omit }}"
@@ -20,7 +22,9 @@ openshift_storage_glusterfs_heketi_url: "{{ omit }}"
openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}"
openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}"
openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}"
-openshift_storage_glusterfs_registry_nodeselector: 'storagenode=registry'
+openshift_storage_glusterfs_registry_name: 'registry'
+openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host"
+openshift_storage_glusterfs_registry_storageclass: False
openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}"
openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}"
openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}"
@@ -29,8 +33,8 @@ openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_gl
openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}"
openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}"
openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}"
-openshift_storage_glusterfs_registry_heketi_admin_key: "{{ openshift_storage_glusterfs_heketi_admin_key }}"
-openshift_storage_glusterfs_registry_heketi_user_key: "{{ openshift_storage_glusterfs_heketi_user_key }}"
+openshift_storage_glusterfs_registry_heketi_admin_key: "{{ 32 | oo_generate_secret }}"
+openshift_storage_glusterfs_registry_heketi_user_key: "{{ 32 | oo_generate_secret }}"
openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}"
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
index c9945be13..81b4fa5dc 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml
@@ -9,49 +9,47 @@ metadata:
annotations:
description: Bootstrap Heketi installation
tags: glusterfs,heketi,installation
-labels:
- template: deploy-heketi
objects:
- kind: Service
apiVersion: v1
metadata:
- name: deploy-heketi
+ name: deploy-heketi-${CLUSTER_NAME}
labels:
- glusterfs: deploy-heketi-service
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-service
deploy-heketi: support
annotations:
description: Exposes Heketi service
spec:
ports:
- - name: deploy-heketi
+ - name: deploy-heketi-${CLUSTER_NAME}
port: 8080
targetPort: 8080
selector:
- name: deploy-heketi
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
- kind: Route
apiVersion: v1
metadata:
- name: deploy-heketi
+ name: deploy-heketi-${CLUSTER_NAME}
labels:
- glusterfs: deploy-heketi-route
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-route
deploy-heketi: support
spec:
to:
kind: Service
- name: deploy-heketi
+ name: deploy-heketi-${CLUSTER_NAME}
- kind: DeploymentConfig
apiVersion: v1
metadata:
- name: deploy-heketi
+ name: deploy-heketi-${CLUSTER_NAME}
labels:
- glusterfs: deploy-heketi-dc
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-dc
deploy-heketi: support
annotations:
description: Defines how to deploy Heketi
spec:
replicas: 1
selector:
- name: deploy-heketi
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
triggers:
- type: ConfigChange
strategy:
@@ -60,13 +58,12 @@ objects:
metadata:
name: deploy-heketi
labels:
- name: deploy-heketi
- glusterfs: deploy-heketi-pod
+ glusterfs: deploy-heketi-${CLUSTER_NAME}-pod
deploy-heketi: support
spec:
- serviceAccountName: heketi-service-account
+ serviceAccountName: heketi-${CLUSTER_NAME}-service-account
containers:
- - name: deploy-heketi
+ - name: heketi
image: ${IMAGE_NAME}:${IMAGE_VERSION}
env:
- name: HEKETI_USER_KEY
@@ -81,11 +78,15 @@ objects:
value: '14'
- name: HEKETI_KUBE_GLUSTER_DAEMONSET
value: '1'
+ - name: HEKETI_KUBE_NAMESPACE
+ value: ${HEKETI_KUBE_NAMESPACE}
ports:
- containerPort: 8080
volumeMounts:
- name: db
mountPath: /var/lib/heketi
+ - name: topology
+ mountPath: ${TOPOLOGY_PATH}
readinessProbe:
timeoutSeconds: 3
initialDelaySeconds: 3
@@ -100,6 +101,9 @@ objects:
port: 8080
volumes:
- name: db
+ - name: topology
+ secret:
+ secretName: heketi-${CLUSTER_NAME}-topology-secret
parameters:
- name: HEKETI_USER_KEY
displayName: Heketi User Secret
@@ -107,9 +111,19 @@ parameters:
- name: HEKETI_ADMIN_KEY
displayName: Heketi Administrator Secret
description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_KUBE_NAMESPACE
+ displayName: Namespace
+ description: Set the namespace where the GlusterFS pods reside
+ value: default
- name: IMAGE_NAME
- displayName: GlusterFS container name
+ displayName: heketi container name
required: True
- name: IMAGE_VERSION
- displayName: GlusterFS container versiona
+ displayName: heketi container versiona
+ required: True
+- name: CLUSTER_NAME
+ displayName: GlusterFS cluster name
+ value: glusterfs
+- name: TOPOLOGY_PATH
+ displayName: heketi topology file location
required: True
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
index c66705752..dc3d2250a 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml
@@ -12,24 +12,24 @@ objects:
- kind: DaemonSet
apiVersion: extensions/v1beta1
metadata:
- name: glusterfs
+ name: glusterfs-${CLUSTER_NAME}
labels:
- glusterfs: daemonset
+ glusterfs: ${CLUSTER_NAME}-daemonset
annotations:
description: GlusterFS DaemonSet
tags: glusterfs
spec:
selector:
matchLabels:
- glusterfs-node: pod
+ glusterfs: ${CLUSTER_NAME}-pod
template:
metadata:
- name: glusterfs
+ name: glusterfs-${CLUSTER_NAME}
labels:
+ glusterfs: ${CLUSTER_NAME}-pod
glusterfs-node: pod
spec:
- nodeSelector:
- storagenode: glusterfs
+ nodeSelector: "${{NODE_LABELS}}"
hostNetwork: true
containers:
- name: glusterfs
@@ -63,26 +63,26 @@ objects:
privileged: true
readinessProbe:
timeoutSeconds: 3
- initialDelaySeconds: 100
+ initialDelaySeconds: 40
exec:
command:
- "/bin/bash"
- "-c"
- systemctl status glusterd.service
- periodSeconds: 10
+ periodSeconds: 25
successThreshold: 1
- failureThreshold: 3
+ failureThreshold: 15
livenessProbe:
timeoutSeconds: 3
- initialDelaySeconds: 100
+ initialDelaySeconds: 40
exec:
command:
- "/bin/bash"
- "-c"
- systemctl status glusterd.service
- periodSeconds: 10
+ periodSeconds: 25
successThreshold: 1
- failureThreshold: 3
+ failureThreshold: 15
resources: {}
terminationMessagePath: "/dev/termination-log"
volumes:
@@ -120,9 +120,16 @@ objects:
dnsPolicy: ClusterFirst
securityContext: {}
parameters:
+- name: NODE_LABELS
+ displayName: Daemonset Node Labels
+ description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=<CLUSTER_NAME>-host\'
+ value: '{ "glusterfs": "storage-host" }'
- name: IMAGE_NAME
displayName: GlusterFS container name
required: True
- name: IMAGE_VERSION
displayName: GlusterFS container versiona
required: True
+- name: CLUSTER_NAME
+ displayName: GlusterFS cluster name
+ value: storage
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
index df045c170..1d8f1abdf 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
+++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml
@@ -8,15 +8,13 @@ metadata:
annotations:
description: Heketi service deployment template
tags: glusterfs,heketi
-labels:
- template: heketi
objects:
- kind: Service
apiVersion: v1
metadata:
- name: heketi
+ name: heketi-${CLUSTER_NAME}
labels:
- glusterfs: heketi-service
+ glusterfs: heketi-${CLUSTER_NAME}-service
annotations:
description: Exposes Heketi service
spec:
@@ -25,40 +23,40 @@ objects:
port: 8080
targetPort: 8080
selector:
- glusterfs: heketi-pod
+ glusterfs: heketi-${CLUSTER_NAME}-pod
- kind: Route
apiVersion: v1
metadata:
- name: heketi
+ name: heketi-${CLUSTER_NAME}
labels:
- glusterfs: heketi-route
+ glusterfs: heketi-${CLUSTER_NAME}-route
spec:
to:
kind: Service
- name: heketi
+ name: heketi-${CLUSTER_NAME}
- kind: DeploymentConfig
apiVersion: v1
metadata:
- name: heketi
+ name: heketi-${CLUSTER_NAME}
labels:
- glusterfs: heketi-dc
+ glusterfs: heketi-${CLUSTER_NAME}-dc
annotations:
description: Defines how to deploy Heketi
spec:
replicas: 1
selector:
- glusterfs: heketi-pod
+ glusterfs: heketi-${CLUSTER_NAME}-pod
triggers:
- type: ConfigChange
strategy:
type: Recreate
template:
metadata:
- name: heketi
+ name: heketi-${CLUSTER_NAME}
labels:
- glusterfs: heketi-pod
+ glusterfs: heketi-${CLUSTER_NAME}-pod
spec:
- serviceAccountName: heketi-service-account
+ serviceAccountName: heketi-${CLUSTER_NAME}-service-account
containers:
- name: heketi
image: ${IMAGE_NAME}:${IMAGE_VERSION}
@@ -76,6 +74,8 @@ objects:
value: '14'
- name: HEKETI_KUBE_GLUSTER_DAEMONSET
value: '1'
+ - name: HEKETI_KUBE_NAMESPACE
+ value: ${HEKETI_KUBE_NAMESPACE}
ports:
- containerPort: 8080
volumeMounts:
@@ -96,7 +96,7 @@ objects:
volumes:
- name: db
glusterfs:
- endpoints: heketi-storage-endpoints
+ endpoints: heketi-db-${CLUSTER_NAME}-endpoints
path: heketidbstorage
parameters:
- name: HEKETI_USER_KEY
@@ -105,9 +105,16 @@ parameters:
- name: HEKETI_ADMIN_KEY
displayName: Heketi Administrator Secret
description: Set secret for administration of the Heketi service as user _admin_
+- name: HEKETI_KUBE_NAMESPACE
+ displayName: Namespace
+ description: Set the namespace where the GlusterFS pods reside
+ value: default
- name: IMAGE_NAME
- displayName: GlusterFS container name
+ displayName: heketi container name
required: True
- name: IMAGE_VERSION
- displayName: GlusterFS container versiona
+ displayName: heketi container versiona
required: True
+- name: CLUSTER_NAME
+ displayName: GlusterFS cluster name
+ value: glusterfs
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
index fa5fa2cb0..829c1f51b 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml
@@ -5,12 +5,6 @@
name: "{{ glusterfs_namespace }}"
when: glusterfs_is_native or glusterfs_heketi_is_native
-- include: glusterfs_deploy.yml
- when: glusterfs_is_native
-
-- name: Make sure heketi-client is installed
- package: name=heketi-client state=present
-
- name: Delete pre-existing heketi resources
oc_obj:
namespace: "{{ glusterfs_namespace }}"
@@ -21,12 +15,18 @@
with_items:
- kind: "template,route,service,dc,jobs,secret"
selector: "deploy-heketi"
- - kind: "template,route,service,dc"
- name: "heketi"
- - kind: "svc,ep"
+ - kind: "svc"
name: "heketi-storage-endpoints"
+ - kind: "secret"
+ name: "heketi-{{ glusterfs_name }}-topology-secret"
+ - kind: "template,route,service,dc"
+ name: "heketi-{{ glusterfs_name }}"
+ - kind: "svc"
+ name: "heketi-db-{{ glusterfs_name }}-endpoints"
- kind: "sa"
- name: "heketi-service-account"
+ name: "heketi-{{ glusterfs_name }}-service-account"
+ - kind: "secret"
+ name: "heketi-{{ glusterfs_name }}-user-secret"
failed_when: False
when: glusterfs_heketi_wipe
@@ -35,11 +35,11 @@
namespace: "{{ glusterfs_namespace }}"
kind: pod
state: list
- selector: "glusterfs=deploy-heketi-pod"
+ selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
until: "heketi_pod.results.results[0]['items'] | count == 0"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
when: glusterfs_heketi_wipe
- name: Wait for heketi pods to terminate
@@ -47,23 +47,26 @@
namespace: "{{ glusterfs_namespace }}"
kind: pod
state: list
- selector: "glusterfs=heketi-pod"
+ selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
until: "heketi_pod.results.results[0]['items'] | count == 0"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
when: glusterfs_heketi_wipe
+- include: glusterfs_deploy.yml
+ when: glusterfs_is_native
+
- name: Create heketi service account
oc_serviceaccount:
namespace: "{{ glusterfs_namespace }}"
- name: heketi-service-account
+ name: "heketi-{{ glusterfs_name }}-service-account"
state: present
when: glusterfs_heketi_is_native
- name: Add heketi service account to privileged SCC
oc_adm_policy_user:
- user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+ user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
resource_kind: scc
resource_name: privileged
state: present
@@ -71,7 +74,7 @@
- name: Allow heketi service account to view/edit pods
oc_adm_policy_user:
- user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-service-account"
+ user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account"
resource_kind: role
resource_name: edit
state: present
@@ -82,7 +85,7 @@
namespace: "{{ glusterfs_namespace }}"
state: list
kind: pod
- selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+ selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
when: glusterfs_heketi_is_native
@@ -100,7 +103,7 @@
namespace: "{{ glusterfs_namespace }}"
state: list
kind: pod
- selector: "glusterfs=heketi-pod"
+ selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
when: glusterfs_heketi_is_native
@@ -113,48 +116,35 @@
# heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True
- "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0"
+- name: Generate topology file
+ template:
+ src: "{{ openshift.common.examples_content_version }}/topology.json.j2"
+ dest: "{{ mktemp.stdout }}/topology.json"
+ when:
+ - glusterfs_heketi_topology_load
+
- include: heketi_deploy_part1.yml
when:
- glusterfs_heketi_is_native
- glusterfs_heketi_deploy_is_missing
- glusterfs_heketi_is_missing
-- name: Determine heketi URL
- oc_obj:
- namespace: "{{ glusterfs_namespace }}"
- state: list
- kind: ep
- selector: "glusterfs in (deploy-heketi-service, heketi-service)"
- register: heketi_url
- until:
- - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
- - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
- delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
- when:
- - glusterfs_heketi_is_native
- - glusterfs_heketi_url is undefined
-
- name: Set heketi URL
set_fact:
- glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+ glusterfs_heketi_url: "localhost:8080"
when:
- glusterfs_heketi_is_native
- - glusterfs_heketi_url is undefined
+
+- name: Set heketi-cli command
+ set_fact:
+ glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
- name: Verify heketi service
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list"
+ command: "{{ glusterfs_heketi_client }} cluster list"
changed_when: False
-- name: Generate topology file
- template:
- src: "{{ openshift.common.examples_content_version }}/topology.json.j2"
- dest: "{{ mktemp.stdout }}/topology.json"
- when:
- - glusterfs_heketi_topology_load
-
- name: Load heketi topology
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
+ command: "{{ glusterfs_heketi_client }} topology load --json={{ mktemp.stdout }}/topology.json 2>&1"
register: topology_load
failed_when: "topology_load.rc != 0 or 'Unable' in topology_load.stdout"
when:
@@ -164,3 +154,29 @@
when:
- glusterfs_heketi_is_native
- glusterfs_heketi_is_missing
+
+- name: Create heketi user secret
+ oc_secret:
+ namespace: "{{ glusterfs_namespace }}"
+ state: present
+ name: "heketi-{{ glusterfs_name }}-user-secret"
+ type: "kubernetes.io/glusterfs"
+ force: True
+ contents:
+ - path: key
+ data: "{{ glusterfs_heketi_user_key }}"
+
+- name: Generate GlusterFS StorageClass file
+ template:
+ src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2"
+ dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml"
+
+- name: Create GlusterFS StorageClass
+ oc_obj:
+ state: present
+ kind: storageclass
+ name: "glusterfs-{{ glusterfs_name }}"
+ files:
+ - "{{ mktemp.stdout }}/glusterfs-storageclass.yml"
+ when:
+ - glusterfs_storageclass
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
index 451990240..aa303d126 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml
@@ -3,7 +3,9 @@
glusterfs_timeout: "{{ openshift_storage_glusterfs_timeout }}"
glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}"
glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}"
- glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | map_from_pairs }}"
+ glusterfs_name: "{{ openshift_storage_glusterfs_name }}"
+ glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}"
+ glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}"
glusterfs_image: "{{ openshift_storage_glusterfs_image }}"
glusterfs_version: "{{ openshift_storage_glusterfs_version }}"
glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}"
@@ -17,6 +19,6 @@
glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}"
glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}"
glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}"
- glusterfs_nodes: "{{ g_glusterfs_hosts }}"
+ glusterfs_nodes: "{{ groups.glusterfs }}"
- include: glusterfs_common.yml
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml
index 579112349..ea4dcc510 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml
@@ -1,23 +1,24 @@
---
- assert:
- that: "glusterfs_nodeselector.keys() | count == 1"
- msg: Only one GlusterFS nodeselector key pair should be provided
-
-- assert:
that: "glusterfs_nodes | count >= 3"
msg: There must be at least three GlusterFS nodes specified
- name: Delete pre-existing GlusterFS resources
oc_obj:
namespace: "{{ glusterfs_namespace }}"
- kind: "template,daemonset"
- name: glusterfs
+ kind: "{{ item.kind }}"
+ name: "{{ item.name }}"
state: absent
+ with_items:
+ - kind: template
+ name: glusterfs
+ - kind: daemonset
+ name: "glusterfs-{{ glusterfs_name }}"
when: glusterfs_wipe
- name: Unlabel any existing GlusterFS nodes
oc_label:
- name: "{{ item }}"
+ name: "{{ hostvars[item].openshift.common.hostname }}"
kind: node
state: absent
labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
@@ -40,11 +41,16 @@
failed_when: False
when: glusterfs_wipe
- # Runs "vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume.
+ # Runs "lvremove -ff <vg>; vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume.
- name: Clear GlusterFS storage device contents
- shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}"
+ shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}lvremove -ff {{ fields[1] }}; vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}"
delegate_to: "{{ item.item }}"
with_items: "{{ devices_info.results }}"
+ register: clear_devices
+ until:
+ - "'contains a filesystem in use' not in clear_devices.stderr"
+ delay: 1
+ retries: 30
when:
- glusterfs_wipe
- item.stdout_lines | count > 0
@@ -61,13 +67,11 @@
- name: Label GlusterFS nodes
oc_label:
- name: "{{ glusterfs_host }}"
+ name: "{{ hostvars[item].openshift.common.hostname }}"
kind: node
state: add
labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}"
with_items: "{{ glusterfs_nodes | default([]) }}"
- loop_control:
- loop_var: glusterfs_host
- name: Copy GlusterFS DaemonSet template
copy:
@@ -78,7 +82,7 @@
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: template
- name: glusterfs
+ name: "glusterfs"
state: present
files:
- "{{ mktemp.stdout }}/glusterfs-template.yml"
@@ -91,17 +95,19 @@
params:
IMAGE_NAME: "{{ glusterfs_image }}"
IMAGE_VERSION: "{{ glusterfs_version }}"
+ NODE_LABELS: "{{ glusterfs_nodeselector }}"
+ CLUSTER_NAME: "{{ glusterfs_name }}"
- name: Wait for GlusterFS pods
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: pod
state: list
- selector: "glusterfs-node=pod"
+ selector: "glusterfs={{ glusterfs_name }}-pod"
register: glusterfs_pods
until:
- "glusterfs_pods.results.results[0]['items'] | count > 0"
# There must be as many pods with 'Ready' staus True as there are nodes expecting those pods
- "glusterfs_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == glusterfs_nodes | count"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
index 392f4b65b..4c6891eeb 100644
--- a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
+++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml
@@ -3,7 +3,9 @@
glusterfs_timeout: "{{ openshift_storage_glusterfs_registry_timeout }}"
glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}"
glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}"
- glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | map_from_pairs }}"
+ glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}"
+ glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}"
+ glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}"
glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}"
glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}"
glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}"
@@ -17,21 +19,22 @@
glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_registry_heketi_topology_load }}"
glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}"
glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}"
- glusterfs_nodes: "{{ g_glusterfs_registry_hosts }}"
+ glusterfs_nodes: "{{ groups.glusterfs_registry }}"
- include: glusterfs_common.yml
- when: g_glusterfs_registry_hosts != g_glusterfs_hosts
+ when:
+ - groups.glusterfs_registry | default([]) | count > 0
+ - "'glusterfs' not in groups or groups.glusterfs_registry != groups.glusterfs"
- name: Delete pre-existing GlusterFS registry resources
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: "{{ item.kind }}"
- name: "{{ item.name | default(omit) }}"
- selector: "{{ item.selector | default(omit) }}"
+ name: "{{ item.name }}"
state: absent
with_items:
- - kind: "svc,ep"
- name: "glusterfs-registry-endpoints"
+ - kind: "svc"
+ name: "glusterfs-{{ glusterfs_name }}-endpoints"
failed_when: False
- name: Generate GlusterFS registry endpoints
@@ -40,8 +43,8 @@
dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
- name: Copy GlusterFS registry service
- copy:
- src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml"
+ template:
+ src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2"
dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
- name: Create GlusterFS registry endpoints
@@ -49,7 +52,7 @@
namespace: "{{ glusterfs_namespace }}"
state: present
kind: endpoints
- name: glusterfs-registry-endpoints
+ name: "glusterfs-{{ glusterfs_name }}-endpoints"
files:
- "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml"
@@ -58,14 +61,14 @@
namespace: "{{ glusterfs_namespace }}"
state: present
kind: service
- name: glusterfs-registry-endpoints
+ name: "glusterfs-{{ glusterfs_name }}-endpoints"
files:
- "{{ mktemp.stdout }}/glusterfs-registry-service.yml"
- name: Check if GlusterFS registry volume exists
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume list"
+ command: "{{ glusterfs_heketi_client }} volume list"
register: registry_volume
- name: Create GlusterFS registry volume
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}"
+ command: "{{ glusterfs_heketi_client }} volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}"
when: "openshift.hosted.registry.storage.glusterfs.path not in registry_volume.stdout"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
index c14fcfb15..318d34b5d 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml
@@ -6,11 +6,21 @@
with_items:
- "deploy-heketi-template.yml"
-- name: Create deploy-heketi resources
+- name: Create heketi topology secret
+ oc_secret:
+ namespace: "{{ glusterfs_namespace }}"
+ state: present
+ name: "heketi-{{ glusterfs_name }}-topology-secret"
+ force: True
+ files:
+ - name: topology.json
+ path: "{{ mktemp.stdout }}/topology.json"
+
+- name: Create deploy-heketi template
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: template
- name: deploy-heketi
+ name: "deploy-heketi"
state: present
files:
- "{{ mktemp.stdout }}/deploy-heketi-template.yml"
@@ -25,17 +35,20 @@
IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+ HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+ CLUSTER_NAME: "{{ glusterfs_name }}"
+ TOPOLOGY_PATH: "{{ mktemp.stdout }}"
- name: Wait for deploy-heketi pod
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: pod
state: list
- selector: "glusterfs=deploy-heketi-pod,deploy-heketi=support"
+ selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
until:
- "heketi_pod.results.results[0]['items'] | count > 0"
# Pod's 'Ready' status must be True
- "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
index 64410a9ab..3a9619d9d 100644
--- a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
+++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml
@@ -1,8 +1,10 @@
---
- name: Create heketi DB volume
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' setup-openshift-heketi-storage --listfile {{ mktemp.stdout }}/heketi-storage.json"
+ command: "{{ glusterfs_heketi_client }} setup-openshift-heketi-storage --listfile /tmp/heketi-storage.json"
register: setup_storage
- failed_when: False
+
+- name: Copy heketi-storage list
+ shell: "{{ openshift.common.client_binary }} rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json"
# This is used in the subsequent task
- name: Copy the admin client config
@@ -28,7 +30,7 @@
# Pod's 'Complete' status must be True
- "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Complete'}) | map('bool') | select | list | count == 1"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
failed_when:
- "'results' in heketi_job.results"
- "heketi_job.results.results | count > 0"
@@ -46,14 +48,45 @@
with_items:
- kind: "template,route,service,jobs,dc,secret"
selector: "deploy-heketi"
- failed_when: False
+ - kind: "svc"
+ name: "heketi-storage-endpoints"
+ - kind: "secret"
+ name: "heketi-{{ glusterfs_name }}-topology-secret"
+
+- name: Generate heketi endpoints
+ template:
+ src: "{{ openshift.common.examples_content_version }}/heketi-endpoints.yml.j2"
+ dest: "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Generate heketi service
+ template:
+ src: "{{ openshift.common.examples_content_version }}/heketi-service.yml.j2"
+ dest: "{{ mktemp.stdout }}/heketi-service.yml"
+
+- name: Create heketi endpoints
+ oc_obj:
+ namespace: "{{ glusterfs_namespace }}"
+ state: present
+ kind: endpoints
+ name: "heketi-db-{{ glusterfs_name }}-endpoints"
+ files:
+ - "{{ mktemp.stdout }}/heketi-endpoints.yml"
+
+- name: Create heketi service
+ oc_obj:
+ namespace: "{{ glusterfs_namespace }}"
+ state: present
+ kind: service
+ name: "heketi-db-{{ glusterfs_name }}-endpoints"
+ files:
+ - "{{ mktemp.stdout }}/heketi-service.yml"
- name: Copy heketi template
copy:
src: "{{ openshift.common.examples_content_version }}/heketi-template.yml"
dest: "{{ mktemp.stdout }}/heketi-template.yml"
-- name: Create heketi resources
+- name: Create heketi template
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: template
@@ -72,38 +105,27 @@
IMAGE_VERSION: "{{ glusterfs_heketi_version }}"
HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}"
HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}"
+ HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}"
+ CLUSTER_NAME: "{{ glusterfs_name }}"
- name: Wait for heketi pod
oc_obj:
namespace: "{{ glusterfs_namespace }}"
kind: pod
state: list
- selector: "glusterfs=heketi-pod"
+ selector: "glusterfs=heketi-{{ glusterfs_name }}-pod"
register: heketi_pod
until:
- "heketi_pod.results.results[0]['items'] | count > 0"
# Pod's 'Ready' status must be True
- "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1"
delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
-
-- name: Determine heketi URL
- oc_obj:
- namespace: "{{ glusterfs_namespace }}"
- state: list
- kind: ep
- selector: "glusterfs=heketi-service"
- register: heketi_url
- until:
- - "heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip != ''"
- - "heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port != ''"
- delay: 10
- retries: "{{ (glusterfs_timeout / 10) | int }}"
+ retries: "{{ (glusterfs_timeout | int / 10) | int }}"
-- name: Set heketi URL
+- name: Set heketi-cli command
set_fact:
- glusterfs_heketi_url: "{{ heketi_url.results.results[0]['items'][0].subsets[0].addresses[0].ip }}:{{ heketi_url.results.results[0]['items'][0].subsets[0].ports[0].port }}"
+ glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}oc rsh {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'"
- name: Verify heketi service
- command: "heketi-cli -s http://{{ glusterfs_heketi_url }} --user admin --secret '{{ glusterfs_heketi_admin_key }}' cluster list"
+ command: "{{ glusterfs_heketi_client }} cluster list"
changed_when: False
diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml
index ebd8db453..c9bfdd1cd 100644
--- a/roles/openshift_storage_glusterfs/tasks/main.yml
+++ b/roles/openshift_storage_glusterfs/tasks/main.yml
@@ -7,12 +7,11 @@
- include: glusterfs_config.yml
when:
- - g_glusterfs_hosts | default([]) | count > 0
+ - groups.glusterfs | default([]) | count > 0
- include: glusterfs_registry.yml
when:
- - g_glusterfs_registry_hosts | default([]) | count > 0
- - "openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap"
+ - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.glusterfs.swap"
- name: Delete temp directory
file:
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2
index 605627ab5..11c9195bb 100644
--- a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2
@@ -1,7 +1,8 @@
+---
apiVersion: v1
kind: Endpoints
metadata:
- name: glusterfs-registry-endpoints
+ name: glusterfs-{{ glusterfs_name }}-endpoints
subsets:
- addresses:
{% for node in glusterfs_nodes %}
diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2
index 3f8d8f507..3f869d2b7 100644
--- a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-registry-service.yml
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2
@@ -2,7 +2,7 @@
apiVersion: v1
kind: Service
metadata:
- name: glusterfs-registry-endpoints
+ name: glusterfs-{{ glusterfs_name }}-endpoints
spec:
ports:
- port: 1
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
new file mode 100644
index 000000000..9b8fae310
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: glusterfs-{{ glusterfs_name }}
+provisioner: kubernetes.io/glusterfs
+parameters:
+ resturl: "http://{{ glusterfs_heketi_url }}:8081"
+ secretNamespace: "{{ glusterfs_namespace }}"
+ secretName: "heketi-{{ glusterfs_name }}-user-secret"
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2
new file mode 100644
index 000000000..99cbdf748
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2
@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+ name: heketi-db-{{ glusterfs_name }}-endpoints
+subsets:
+- addresses:
+{% for node in glusterfs_nodes %}
+ - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}
+{% endfor %}
+ ports:
+ - port: 1
diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2
new file mode 100644
index 000000000..dcb896441
--- /dev/null
+++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: heketi-db-{{ glusterfs_name }}-endpoints
+spec:
+ ports:
+ - port: 1
+status:
+ loadBalancer: {}