diff options
Diffstat (limited to 'roles/openshift_storage_glusterfs')
20 files changed, 1357 insertions, 0 deletions
diff --git a/roles/openshift_storage_glusterfs/README.md b/roles/openshift_storage_glusterfs/README.md new file mode 100644 index 000000000..da4e348b4 --- /dev/null +++ b/roles/openshift_storage_glusterfs/README.md @@ -0,0 +1,150 @@ +OpenShift GlusterFS Cluster +=========================== + +OpenShift GlusterFS Cluster Configuration + +This role handles the configuration of GlusterFS clusters. It can handle +two primary configuration scenarios: + +* Configuring a new, natively-hosted GlusterFS cluster. In this scenario, + GlusterFS pods are deployed on nodes in the OpenShift cluster which are + configured to provide storage. +* Configuring a new, external GlusterFS cluster. In this scenario, the + cluster nodes have the GlusterFS software pre-installed but have not + been configured yet. The installer will take care of configuring the + cluster(s) for use by OpenShift applications. +* Using existing GlusterFS clusters. In this scenario, one or more + GlusterFS clusters are assumed to be already setup. These clusters can + be either natively-hosted or external, but must be managed by a + [heketi service](https://github.com/heketi/heketi). + +As part of the configuration, a particular GlusterFS cluster may be +specified to provide backend storage for a natively-hosted Docker +registry. + +Unless configured otherwise, a StorageClass will be automatically +created for each non-registry GlusterFS cluster. This will allow +applications which can mount PersistentVolumes to request +dynamically-provisioned GlusterFS volumes. + +Requirements +------------ + +* Ansible 2.2 + +Host Groups +----------- + +The following group is expected to be populated for this role to run: + +* `[glusterfs]` + +Additionally, the following group may be specified either in addition to or +instead of the above group to deploy a GlusterFS cluster for use by a natively +hosted Docker registry: + +* `[glusterfs_registry]` + +Host Variables +-------------- + +For configuring new clusters, the following role variables are available. + +Each host in either of the above groups must have the following variable +defined: + +| Name | Default value | Description | +|-------------------|---------------|-----------------------------------------| +| glusterfs_devices | None | A list of block devices that will be completely managed as part of a GlusterFS cluster. There must be at least one device listed. Each device must be bare, e.g. no partitions or LVM PVs. **Example:** '[ "/dev/sdb" ]' + +In addition, each host may specify the following variables to further control +their configuration as GlusterFS nodes: + +| Name | Default value | Description | +|--------------------|---------------------------|-----------------------------------------| +| glusterfs_cluster | 1 | The ID of the cluster this node should belong to. This is useful when a single heketi service is expected to manage multiple distinct clusters. **NOTE:** For natively-hosted clusters, all pods will be in the same OpenShift namespace +| glusterfs_hostname | openshift.common.hostname | A hostname (or IP address) that will be used for internal GlusterFS communication +| glusterfs_ip | openshift.common.ip | An IP address that will be used by pods to communicate with the GlusterFS node +| glusterfs_zone | 1 | A zone number for the node. Zones are used within the cluster for determining how to distribute the bricks of GlusterFS volumes. heketi will try to spread each volumes' bricks as evenly as possible across all zones + +Role Variables +-------------- + +This role has the following variables that control the integration of a +GlusterFS cluster into a new or existing OpenShift cluster: + +| Name | Default value | Description | +|--------------------------------------------------|-------------------------|-----------------------------------------| +| openshift_storage_glusterfs_timeout | 300 | Seconds to wait for pods to become ready +| openshift_storage_glusterfs_namespace | 'default' | Namespace in which to create GlusterFS resources +| openshift_storage_glusterfs_is_native | True | GlusterFS should be containerized +| openshift_storage_glusterfs_name | 'storage' | A name to identify the GlusterFS cluster, which will be used in resource names +| openshift_storage_glusterfs_nodeselector | 'glusterfs=storage-host'| Selector to determine which nodes will host GlusterFS pods in native mode. **NOTE:** The label value is taken from the cluster name +| openshift_storage_glusterfs_storageclass | True | Automatically create a StorageClass for each GlusterFS cluster +| openshift_storage_glusterfs_image | 'gluster/gluster-centos'| Container image to use for GlusterFS pods, enterprise default is 'rhgs3/rhgs-server-rhel7' +| openshift_storage_glusterfs_version | 'latest' | Container image version to use for GlusterFS pods +| openshift_storage_glusterfs_wipe | False | Destroy any existing GlusterFS resources and wipe storage devices. **WARNING: THIS WILL DESTROY ANY DATA ON THOSE DEVICES.** +| openshift_storage_glusterfs_heketi_is_native | True | heketi should be containerized +| openshift_storage_glusterfs_heketi_image | 'heketi/heketi' | Container image to use for heketi pods, enterprise default is 'rhgs3/rhgs-volmanager-rhel7' +| openshift_storage_glusterfs_heketi_version | 'latest' | Container image version to use for heketi pods +| openshift_storage_glusterfs_heketi_admin_key | auto-generated | String to use as secret key for performing heketi commands as admin +| openshift_storage_glusterfs_heketi_user_key | auto-generated | String to use as secret key for performing heketi commands as user that can only view or modify volumes +| openshift_storage_glusterfs_heketi_topology_load | True | Load the GlusterFS topology information into heketi +| openshift_storage_glusterfs_heketi_url | Undefined | When heketi is native, this sets the hostname portion of the final heketi route URL. When heketi is external, this is the full URL to the heketi service. +| openshift_storage_glusterfs_heketi_port | 8080 | TCP port for external heketi service **NOTE:** This has no effect in native mode +| openshift_storage_glusterfs_heketi_wipe | False | Destroy any existing heketi resources, defaults to the value of `openshift_storage_glusterfs_wipe` + +Each role variable also has a corresponding variable to optionally configure a +separate GlusterFS cluster for use as storage for an integrated Docker +registry. These variables start with the prefix +`openshift_storage_glusterfs_registry_` and, for the most part, default to the +values in their corresponding non-registry variables. The following variables +are an exception: + +| Name | Default value | Description | +|-------------------------------------------------------|-----------------------|-----------------------------------------| +| openshift_storage_glusterfs_registry_namespace | registry namespace | Default is to use the hosted registry's namespace, otherwise 'default' +| openshift_storage_glusterfs_registry_name | 'registry' | This allows for the logical separation of the registry GlusterFS cluster from other GlusterFS clusters +| openshift_storage_glusterfs_registry_storageclass | False | It is recommended to not create a StorageClass for GlusterFS clusters serving registry storage, so as to avoid performance penalties +| openshift_storage_glusterfs_registry_heketi_admin_key | auto-generated | Separate from the above +| openshift_storage_glusterfs_registry_heketi_user_key | auto-generated | Separate from the above + +Additionally, this role's behavior responds to the following registry-specific +variables: + +| Name | Default value | Description | +|-----------------------------------------------|------------------------------|-----------------------------------------| +| openshift_hosted_registry_glusterfs_endpoints | glusterfs-registry-endpoints | The name for the Endpoints resource that will point the registry to the GlusterFS nodes +| openshift_hosted_registry_glusterfs_path | glusterfs-registry-volume | The name for the GlusterFS volume that will provide registry storage +| openshift_hosted_registry_glusterfs_readonly | False | Whether the GlusterFS volume should be read-only +| openshift_hosted_registry_glusterfs_swap | False | Whether to swap an existing registry's storage volume for a GlusterFS volume +| openshift_hosted_registry_glusterfs_swapcopy | True | If swapping, copy the contents of the pre-existing registry storage to the new GlusterFS volume + +Dependencies +------------ + +* os_firewall +* openshift_hosted_facts +* openshift_repos +* lib_openshift + +Example Playbook +---------------- + +``` +- name: Configure GlusterFS hosts + hosts: oo_first_master + roles: + - role: openshift_storage_glusterfs + when: groups.oo_glusterfs_to_config | default([]) | count > 0 +``` + +License +------- + +Apache License, Version 2.0 + +Author Information +------------------ + +Jose A. Rivera (jarrpa@redhat.com) diff --git a/roles/openshift_storage_glusterfs/defaults/main.yml b/roles/openshift_storage_glusterfs/defaults/main.yml new file mode 100644 index 000000000..4ff56af9e --- /dev/null +++ b/roles/openshift_storage_glusterfs/defaults/main.yml @@ -0,0 +1,42 @@ +--- +openshift_storage_glusterfs_timeout: 300 +openshift_storage_glusterfs_namespace: 'default' +openshift_storage_glusterfs_is_native: True +openshift_storage_glusterfs_name: 'storage' +openshift_storage_glusterfs_nodeselector: "glusterfs={{ openshift_storage_glusterfs_name }}-host" +openshift_storage_glusterfs_storageclass: True +openshift_storage_glusterfs_image: "{{ 'rhgs3/rhgs-server-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'gluster/gluster-centos' | quote }}" +openshift_storage_glusterfs_version: 'latest' +openshift_storage_glusterfs_wipe: False +openshift_storage_glusterfs_heketi_is_native: True +openshift_storage_glusterfs_heketi_is_missing: True +openshift_storage_glusterfs_heketi_deploy_is_missing: True +openshift_storage_glusterfs_heketi_image: "{{ 'rhgs3/rhgs-volmanager-rhel7' | quote if deployment_type == 'openshift-enterprise' else 'heketi/heketi' | quote }}" +openshift_storage_glusterfs_heketi_version: 'latest' +openshift_storage_glusterfs_heketi_admin_key: "{{ omit }}" +openshift_storage_glusterfs_heketi_user_key: "{{ omit }}" +openshift_storage_glusterfs_heketi_topology_load: True +openshift_storage_glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_wipe }}" +openshift_storage_glusterfs_heketi_url: "{{ omit }}" +openshift_storage_glusterfs_heketi_port: 8080 + +openshift_storage_glusterfs_registry_timeout: "{{ openshift_storage_glusterfs_timeout }}" +openshift_storage_glusterfs_registry_namespace: "{{ openshift.hosted.registry.namespace | default('default') }}" +openshift_storage_glusterfs_registry_is_native: "{{ openshift_storage_glusterfs_is_native }}" +openshift_storage_glusterfs_registry_name: 'registry' +openshift_storage_glusterfs_registry_nodeselector: "glusterfs={{ openshift_storage_glusterfs_registry_name }}-host" +openshift_storage_glusterfs_registry_storageclass: False +openshift_storage_glusterfs_registry_image: "{{ openshift_storage_glusterfs_image }}" +openshift_storage_glusterfs_registry_version: "{{ openshift_storage_glusterfs_version }}" +openshift_storage_glusterfs_registry_wipe: "{{ openshift_storage_glusterfs_wipe }}" +openshift_storage_glusterfs_registry_heketi_is_native: "{{ openshift_storage_glusterfs_heketi_is_native }}" +openshift_storage_glusterfs_registry_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing }}" +openshift_storage_glusterfs_registry_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}" +openshift_storage_glusterfs_registry_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}" +openshift_storage_glusterfs_registry_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}" +openshift_storage_glusterfs_registry_heketi_admin_key: "{{ omit }}" +openshift_storage_glusterfs_registry_heketi_user_key: "{{ omit }}" +openshift_storage_glusterfs_registry_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}" +openshift_storage_glusterfs_registry_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}" +openshift_storage_glusterfs_registry_heketi_url: "{{ openshift_storage_glusterfs_heketi_url | default(omit) }}" +openshift_storage_glusterfs_registry_heketi_port: 8080 diff --git a/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml new file mode 100644 index 000000000..4434f750c --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.6/deploy-heketi-template.yml @@ -0,0 +1,134 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: deploy-heketi + labels: + glusterfs: heketi-template + deploy-heketi: support + annotations: + description: Bootstrap Heketi installation + tags: glusterfs,heketi,installation +objects: +- kind: Service + apiVersion: v1 + metadata: + name: deploy-heketi-${CLUSTER_NAME} + labels: + glusterfs: deploy-heketi-${CLUSTER_NAME}-service + deploy-heketi: support + annotations: + description: Exposes Heketi service + spec: + ports: + - name: deploy-heketi-${CLUSTER_NAME} + port: 8080 + targetPort: 8080 + selector: + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod +- kind: Route + apiVersion: v1 + metadata: + name: ${HEKETI_ROUTE} + labels: + glusterfs: deploy-heketi-${CLUSTER_NAME}-route + deploy-heketi: support + spec: + to: + kind: Service + name: deploy-heketi-${CLUSTER_NAME} +- kind: DeploymentConfig + apiVersion: v1 + metadata: + name: deploy-heketi-${CLUSTER_NAME} + labels: + glusterfs: deploy-heketi-${CLUSTER_NAME}-dc + deploy-heketi: support + annotations: + description: Defines how to deploy Heketi + spec: + replicas: 1 + selector: + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod + triggers: + - type: ConfigChange + strategy: + type: Recreate + template: + metadata: + name: deploy-heketi + labels: + glusterfs: deploy-heketi-${CLUSTER_NAME}-pod + deploy-heketi: support + spec: + serviceAccountName: heketi-${CLUSTER_NAME}-service-account + containers: + - name: heketi + image: ${IMAGE_NAME}:${IMAGE_VERSION} + env: + - name: HEKETI_USER_KEY + value: ${HEKETI_USER_KEY} + - name: HEKETI_ADMIN_KEY + value: ${HEKETI_ADMIN_KEY} + - name: HEKETI_EXECUTOR + value: kubernetes + - name: HEKETI_FSTAB + value: /var/lib/heketi/fstab + - name: HEKETI_SNAPSHOT_LIMIT + value: '14' + - name: HEKETI_KUBE_GLUSTER_DAEMONSET + value: '1' + - name: HEKETI_KUBE_NAMESPACE + value: ${HEKETI_KUBE_NAMESPACE} + ports: + - containerPort: 8080 + volumeMounts: + - name: db + mountPath: /var/lib/heketi + - name: topology + mountPath: ${TOPOLOGY_PATH} + readinessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 3 + httpGet: + path: /hello + port: 8080 + livenessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 30 + httpGet: + path: /hello + port: 8080 + volumes: + - name: db + - name: topology + secret: + secretName: heketi-${CLUSTER_NAME}-topology-secret +parameters: +- name: HEKETI_USER_KEY + displayName: Heketi User Secret + description: Set secret for those creating volumes as type _user_ +- name: HEKETI_ADMIN_KEY + displayName: Heketi Administrator Secret + description: Set secret for administration of the Heketi service as user _admin_ +- name: HEKETI_KUBE_NAMESPACE + displayName: Namespace + description: Set the namespace where the GlusterFS pods reside + value: default +- name: HEKETI_ROUTE + displayName: heketi route name + description: Set the hostname for the route URL + value: "heketi-glusterfs" +- name: IMAGE_NAME + displayName: heketi container image name + required: True +- name: IMAGE_VERSION + displayName: heketi container image version + required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify this heketi service, useful for running multiple heketi instances + value: glusterfs +- name: TOPOLOGY_PATH + displayName: heketi topology file location + required: True diff --git a/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml new file mode 100644 index 000000000..8c5e1ded3 --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.6/glusterfs-template.yml @@ -0,0 +1,136 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: glusterfs + labels: + glusterfs: template + annotations: + description: GlusterFS DaemonSet template + tags: glusterfs +objects: +- kind: DaemonSet + apiVersion: extensions/v1beta1 + metadata: + name: glusterfs-${CLUSTER_NAME} + labels: + glusterfs: ${CLUSTER_NAME}-daemonset + annotations: + description: GlusterFS DaemonSet + tags: glusterfs + spec: + selector: + matchLabels: + glusterfs: ${CLUSTER_NAME}-pod + template: + metadata: + name: glusterfs-${CLUSTER_NAME} + labels: + glusterfs: ${CLUSTER_NAME}-pod + glusterfs-node: pod + spec: + nodeSelector: "${{NODE_LABELS}}" + hostNetwork: true + containers: + - name: glusterfs + image: ${IMAGE_NAME}:${IMAGE_VERSION} + imagePullPolicy: IfNotPresent + volumeMounts: + - name: glusterfs-heketi + mountPath: "/var/lib/heketi" + - name: glusterfs-run + mountPath: "/run" + - name: glusterfs-lvm + mountPath: "/run/lvm" + - name: glusterfs-etc + mountPath: "/etc/glusterfs" + - name: glusterfs-logs + mountPath: "/var/log/glusterfs" + - name: glusterfs-config + mountPath: "/var/lib/glusterd" + - name: glusterfs-dev + mountPath: "/dev" + - name: glusterfs-misc + mountPath: "/var/lib/misc/glusterfsd" + - name: glusterfs-cgroup + mountPath: "/sys/fs/cgroup" + readOnly: true + - name: glusterfs-ssl + mountPath: "/etc/ssl" + readOnly: true + securityContext: + capabilities: {} + privileged: true + readinessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 40 + exec: + command: + - "/bin/bash" + - "-c" + - systemctl status glusterd.service + periodSeconds: 25 + successThreshold: 1 + failureThreshold: 15 + livenessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 40 + exec: + command: + - "/bin/bash" + - "-c" + - systemctl status glusterd.service + periodSeconds: 25 + successThreshold: 1 + failureThreshold: 15 + resources: {} + terminationMessagePath: "/dev/termination-log" + volumes: + - name: glusterfs-heketi + hostPath: + path: "/var/lib/heketi" + - name: glusterfs-run + emptyDir: {} + - name: glusterfs-lvm + hostPath: + path: "/run/lvm" + - name: glusterfs-etc + hostPath: + path: "/etc/glusterfs" + - name: glusterfs-logs + hostPath: + path: "/var/log/glusterfs" + - name: glusterfs-config + hostPath: + path: "/var/lib/glusterd" + - name: glusterfs-dev + hostPath: + path: "/dev" + - name: glusterfs-misc + hostPath: + path: "/var/lib/misc/glusterfsd" + - name: glusterfs-cgroup + hostPath: + path: "/sys/fs/cgroup" + - name: glusterfs-ssl + hostPath: + path: "/etc/ssl" + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} +parameters: +- name: NODE_LABELS + displayName: Daemonset Node Labels + description: Labels which define the daemonset node selector. Must contain at least one label of the format \'glusterfs=<CLUSTER_NAME>-host\' + value: '{ "glusterfs": "storage-host" }' +- name: IMAGE_NAME + displayName: GlusterFS container image name + required: True +- name: IMAGE_VERSION + displayName: GlusterFS container image version + required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify which heketi service manages this cluster, useful for running multiple heketi instances + value: storage diff --git a/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml new file mode 100644 index 000000000..e3fa0a9fb --- /dev/null +++ b/roles/openshift_storage_glusterfs/files/v3.6/heketi-template.yml @@ -0,0 +1,125 @@ +--- +kind: Template +apiVersion: v1 +metadata: + name: heketi + labels: + glusterfs: heketi-template + annotations: + description: Heketi service deployment template + tags: glusterfs,heketi +objects: +- kind: Service + apiVersion: v1 + metadata: + name: heketi-${CLUSTER_NAME} + labels: + glusterfs: heketi-${CLUSTER_NAME}-service + annotations: + description: Exposes Heketi service + spec: + ports: + - name: heketi + port: 8080 + targetPort: 8080 + selector: + glusterfs: heketi-${CLUSTER_NAME}-pod +- kind: Route + apiVersion: v1 + metadata: + name: ${HEKETI_ROUTE} + labels: + glusterfs: heketi-${CLUSTER_NAME}-route + spec: + to: + kind: Service + name: heketi-${CLUSTER_NAME} +- kind: DeploymentConfig + apiVersion: v1 + metadata: + name: heketi-${CLUSTER_NAME} + labels: + glusterfs: heketi-${CLUSTER_NAME}-dc + annotations: + description: Defines how to deploy Heketi + spec: + replicas: 1 + selector: + glusterfs: heketi-${CLUSTER_NAME}-pod + triggers: + - type: ConfigChange + strategy: + type: Recreate + template: + metadata: + name: heketi-${CLUSTER_NAME} + labels: + glusterfs: heketi-${CLUSTER_NAME}-pod + spec: + serviceAccountName: heketi-${CLUSTER_NAME}-service-account + containers: + - name: heketi + image: ${IMAGE_NAME}:${IMAGE_VERSION} + imagePullPolicy: IfNotPresent + env: + - name: HEKETI_USER_KEY + value: ${HEKETI_USER_KEY} + - name: HEKETI_ADMIN_KEY + value: ${HEKETI_ADMIN_KEY} + - name: HEKETI_EXECUTOR + value: kubernetes + - name: HEKETI_FSTAB + value: /var/lib/heketi/fstab + - name: HEKETI_SNAPSHOT_LIMIT + value: '14' + - name: HEKETI_KUBE_GLUSTER_DAEMONSET + value: '1' + - name: HEKETI_KUBE_NAMESPACE + value: ${HEKETI_KUBE_NAMESPACE} + ports: + - containerPort: 8080 + volumeMounts: + - name: db + mountPath: /var/lib/heketi + readinessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 3 + httpGet: + path: /hello + port: 8080 + livenessProbe: + timeoutSeconds: 3 + initialDelaySeconds: 30 + httpGet: + path: /hello + port: 8080 + volumes: + - name: db + glusterfs: + endpoints: heketi-db-${CLUSTER_NAME}-endpoints + path: heketidbstorage +parameters: +- name: HEKETI_USER_KEY + displayName: Heketi User Secret + description: Set secret for those creating volumes as type _user_ +- name: HEKETI_ADMIN_KEY + displayName: Heketi Administrator Secret + description: Set secret for administration of the Heketi service as user _admin_ +- name: HEKETI_KUBE_NAMESPACE + displayName: Namespace + description: Set the namespace where the GlusterFS pods reside + value: default +- name: HEKETI_ROUTE + displayName: heketi route name + description: Set the hostname for the route URL + value: "heketi-glusterfs" +- name: IMAGE_NAME + displayName: heketi container image name + required: True +- name: IMAGE_VERSION + displayName: heketi container image version + required: True +- name: CLUSTER_NAME + displayName: GlusterFS cluster name + description: A unique name to identify this heketi service, useful for running multiple heketi instances + value: glusterfs diff --git a/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py b/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py new file mode 100644 index 000000000..a86c96df7 --- /dev/null +++ b/roles/openshift_storage_glusterfs/filter_plugins/openshift_storage_glusterfs.py @@ -0,0 +1,23 @@ +''' + Openshift Storage GlusterFS class that provides useful filters used in GlusterFS +''' + + +def map_from_pairs(source, delim="="): + ''' Returns a dict given the source and delim delimited ''' + if source == '': + return dict() + + return dict(item.split(delim) for item in source.split(",")) + + +# pylint: disable=too-few-public-methods +class FilterModule(object): + ''' OpenShift Storage GlusterFS Filters ''' + + # pylint: disable=no-self-use, too-few-public-methods + def filters(self): + ''' Returns the names of the filters provided by this class ''' + return { + 'map_from_pairs': map_from_pairs + } diff --git a/roles/openshift_storage_glusterfs/meta/main.yml b/roles/openshift_storage_glusterfs/meta/main.yml new file mode 100644 index 000000000..aab9851f9 --- /dev/null +++ b/roles/openshift_storage_glusterfs/meta/main.yml @@ -0,0 +1,15 @@ +--- +galaxy_info: + author: Jose A. Rivera + description: OpenShift GlusterFS Cluster + company: Red Hat, Inc. + license: Apache License, Version 2.0 + min_ansible_version: 2.2 + platforms: + - name: EL + versions: + - 7 +dependencies: +- role: openshift_hosted_facts +- role: openshift_repos +- role: lib_openshift diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml new file mode 100644 index 000000000..af901103e --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_common.yml @@ -0,0 +1,217 @@ +--- +- name: Verify target namespace exists + oc_project: + state: present + name: "{{ glusterfs_namespace }}" + when: glusterfs_is_native or glusterfs_heketi_is_native + +- name: Delete pre-existing heketi resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name | default(omit) }}" + selector: "{{ item.selector | default(omit) }}" + state: absent + with_items: + - kind: "template,route,service,dc,jobs,secret" + selector: "deploy-heketi" + - kind: "svc" + name: "heketi-storage-endpoints" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-topology-secret" + - kind: "template,route,service,dc" + name: "heketi-{{ glusterfs_name }}" + - kind: "svc" + name: "heketi-db-{{ glusterfs_name }}-endpoints" + - kind: "sa" + name: "heketi-{{ glusterfs_name }}-service-account" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-admin-secret" + failed_when: False + when: glusterfs_heketi_wipe + +- name: Wait for deploy-heketi pods to terminate + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + until: "heketi_pod.results.results[0]['items'] | count == 0" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + when: glusterfs_heketi_wipe + +- name: Wait for heketi pods to terminate + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + until: "heketi_pod.results.results[0]['items'] | count == 0" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + when: glusterfs_heketi_wipe + +- include: glusterfs_deploy.yml + when: glusterfs_is_native + +- name: Create heketi service account + oc_serviceaccount: + namespace: "{{ glusterfs_namespace }}" + name: "heketi-{{ glusterfs_name }}-service-account" + state: present + when: glusterfs_heketi_is_native + +- name: Add heketi service account to privileged SCC + oc_adm_policy_user: + namespace: "{{ glusterfs_namespace }}" + user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account" + resource_kind: scc + resource_name: privileged + state: present + when: glusterfs_heketi_is_native + +- name: Allow heketi service account to view/edit pods + oc_adm_policy_user: + namespace: "{{ glusterfs_namespace }}" + user: "system:serviceaccount:{{ glusterfs_namespace }}:heketi-{{ glusterfs_name }}-service-account" + resource_kind: role + resource_name: edit + state: present + when: glusterfs_heketi_is_native + +- name: Check for existing deploy-heketi pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: list + kind: pod + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + when: glusterfs_heketi_is_native + +- name: Check if need to deploy deploy-heketi + set_fact: + glusterfs_heketi_deploy_is_missing: False + when: + - "glusterfs_heketi_is_native" + - "heketi_pod.results.results[0]['items'] | count > 0" + # deploy-heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True + - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0" + +- name: Check for existing heketi pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: list + kind: pod + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + when: glusterfs_heketi_is_native + +- name: Check if need to deploy heketi + set_fact: + glusterfs_heketi_is_missing: False + when: + - "glusterfs_heketi_is_native" + - "heketi_pod.results.results[0]['items'] | count > 0" + # heketi is not missing when there are one or more pods with matching labels whose 'Ready' status is True + - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count > 0" + +- name: Generate topology file + template: + src: "{{ openshift.common.examples_content_version }}/topology.json.j2" + dest: "{{ mktemp.stdout }}/topology.json" + when: + - glusterfs_heketi_topology_load + +- name: Generate heketi admin key + set_fact: + glusterfs_heketi_admin_key: "{{ 32 | oo_generate_secret }}" + when: + - glusterfs_heketi_is_native + - glusterfs_heketi_admin_key is undefined + +- name: Generate heketi user key + set_fact: + glusterfs_heketi_user_key: "{{ 32 | oo_generate_secret }}" + until: "glusterfs_heketi_user_key != glusterfs_heketi_admin_key" + delay: 1 + retries: 10 + when: + - glusterfs_heketi_is_native + - glusterfs_heketi_user_key is undefined + +- include: heketi_deploy_part1.yml + when: + - glusterfs_heketi_is_native + - glusterfs_heketi_deploy_is_missing + - glusterfs_heketi_is_missing + +- name: Set heketi-cli command + set_fact: + glusterfs_heketi_client: "{% if glusterfs_heketi_is_native %}{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} {% endif %}heketi-cli -s http://{% if glusterfs_heketi_is_native %}localhost:8080{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %} --user admin --secret '{{ glusterfs_heketi_admin_key }}'" + +- name: Verify heketi service + command: "{{ glusterfs_heketi_client }} cluster list" + changed_when: False + +- name: Load heketi topology + command: "{{ glusterfs_heketi_client }} topology load --json={{ mktemp.stdout }}/topology.json 2>&1" + register: topology_load + failed_when: "topology_load.rc != 0 or 'Unable' in topology_load.stdout" + when: + - glusterfs_heketi_topology_load + +- include: heketi_deploy_part2.yml + when: + - glusterfs_heketi_is_native + - glusterfs_heketi_is_missing + +- name: Create heketi secret + oc_secret: + namespace: "{{ glusterfs_namespace }}" + state: present + name: "heketi-{{ glusterfs_name }}-admin-secret" + type: "kubernetes.io/glusterfs" + force: True + contents: + - path: key + data: "{{ glusterfs_heketi_admin_key }}" + when: + - glusterfs_storageclass + +- name: Get heketi route + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: route + state: list + name: "heketi-{{ glusterfs_name }}" + register: heketi_route + when: + - glusterfs_storageclass + - glusterfs_heketi_is_native + +- name: Determine StorageClass heketi URL + set_fact: + glusterfs_heketi_route: "{{ heketi_route.results.results[0]['spec']['host'] }}" + when: + - glusterfs_storageclass + - glusterfs_heketi_is_native + +- name: Generate GlusterFS StorageClass file + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-storageclass.yml.j2" + dest: "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + when: + - glusterfs_storageclass + +- name: Create GlusterFS StorageClass + oc_obj: + state: present + kind: storageclass + name: "glusterfs-{{ glusterfs_name }}" + files: + - "{{ mktemp.stdout }}/glusterfs-storageclass.yml" + when: + - glusterfs_storageclass diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml new file mode 100644 index 000000000..dbfe126a4 --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_config.yml @@ -0,0 +1,25 @@ +--- +- set_fact: + glusterfs_timeout: "{{ openshift_storage_glusterfs_timeout }}" + glusterfs_namespace: "{{ openshift_storage_glusterfs_namespace }}" + glusterfs_is_native: "{{ openshift_storage_glusterfs_is_native }}" + glusterfs_name: "{{ openshift_storage_glusterfs_name }}" + glusterfs_nodeselector: "{{ openshift_storage_glusterfs_nodeselector | default(['storagenode', openshift_storage_glusterfs_name] | join('=')) | map_from_pairs }}" + glusterfs_storageclass: "{{ openshift_storage_glusterfs_storageclass }}" + glusterfs_image: "{{ openshift_storage_glusterfs_image }}" + glusterfs_version: "{{ openshift_storage_glusterfs_version }}" + glusterfs_wipe: "{{ openshift_storage_glusterfs_wipe }}" + glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_heketi_is_native }}" + glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_heketi_is_missing }}" + glusterfs_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_heketi_deploy_is_missing }}" + glusterfs_heketi_image: "{{ openshift_storage_glusterfs_heketi_image }}" + glusterfs_heketi_version: "{{ openshift_storage_glusterfs_heketi_version }}" + glusterfs_heketi_admin_key: "{{ openshift_storage_glusterfs_heketi_admin_key }}" + glusterfs_heketi_user_key: "{{ openshift_storage_glusterfs_heketi_user_key }}" + glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_heketi_topology_load }}" + glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_heketi_wipe }}" + glusterfs_heketi_url: "{{ openshift_storage_glusterfs_heketi_url }}" + glusterfs_heketi_port: "{{ openshift_storage_glusterfs_heketi_port }}" + glusterfs_nodes: "{{ groups.glusterfs }}" + +- include: glusterfs_common.yml diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml new file mode 100644 index 000000000..ea4dcc510 --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_deploy.yml @@ -0,0 +1,113 @@ +--- +- assert: + that: "glusterfs_nodes | count >= 3" + msg: There must be at least three GlusterFS nodes specified + +- name: Delete pre-existing GlusterFS resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name }}" + state: absent + with_items: + - kind: template + name: glusterfs + - kind: daemonset + name: "glusterfs-{{ glusterfs_name }}" + when: glusterfs_wipe + +- name: Unlabel any existing GlusterFS nodes + oc_label: + name: "{{ hostvars[item].openshift.common.hostname }}" + kind: node + state: absent + labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}" + with_items: "{{ groups.all }}" + when: glusterfs_wipe + +- name: Delete pre-existing GlusterFS config + file: + path: /var/lib/glusterd + state: absent + delegate_to: "{{ item }}" + with_items: "{{ glusterfs_nodes | default([]) }}" + when: glusterfs_wipe + +- name: Get GlusterFS storage devices state + command: "pvdisplay -C --noheadings -o pv_name,vg_name {% for device in hostvars[item].glusterfs_devices %}{{ device }} {% endfor %}" + register: devices_info + delegate_to: "{{ item }}" + with_items: "{{ glusterfs_nodes | default([]) }}" + failed_when: False + when: glusterfs_wipe + + # Runs "lvremove -ff <vg>; vgremove -fy <vg>; pvremove -fy <pv>" for every device found to be a physical volume. +- name: Clear GlusterFS storage device contents + shell: "{% for line in item.stdout_lines %}{% set fields = line.split() %}{% if fields | count > 1 %}lvremove -ff {{ fields[1] }}; vgremove -fy {{ fields[1] }}; {% endif %}pvremove -fy {{ fields[0] }}; {% endfor %}" + delegate_to: "{{ item.item }}" + with_items: "{{ devices_info.results }}" + register: clear_devices + until: + - "'contains a filesystem in use' not in clear_devices.stderr" + delay: 1 + retries: 30 + when: + - glusterfs_wipe + - item.stdout_lines | count > 0 + +- name: Add service accounts to privileged SCC + oc_adm_policy_user: + user: "system:serviceaccount:{{ glusterfs_namespace }}:{{ item }}" + resource_kind: scc + resource_name: privileged + state: present + with_items: + - 'default' + - 'router' + +- name: Label GlusterFS nodes + oc_label: + name: "{{ hostvars[item].openshift.common.hostname }}" + kind: node + state: add + labels: "{{ glusterfs_nodeselector | oo_dict_to_list_of_dict }}" + with_items: "{{ glusterfs_nodes | default([]) }}" + +- name: Copy GlusterFS DaemonSet template + copy: + src: "{{ openshift.common.examples_content_version }}/glusterfs-template.yml" + dest: "{{ mktemp.stdout }}/glusterfs-template.yml" + +- name: Create GlusterFS template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: "glusterfs" + state: present + files: + - "{{ mktemp.stdout }}/glusterfs-template.yml" + +- name: Deploy GlusterFS pods + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "glusterfs" + create: True + params: + IMAGE_NAME: "{{ glusterfs_image }}" + IMAGE_VERSION: "{{ glusterfs_version }}" + NODE_LABELS: "{{ glusterfs_nodeselector }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + +- name: Wait for GlusterFS pods + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs={{ glusterfs_name }}-pod" + register: glusterfs_pods + until: + - "glusterfs_pods.results.results[0]['items'] | count > 0" + # There must be as many pods with 'Ready' staus True as there are nodes expecting those pods + - "glusterfs_pods.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == glusterfs_nodes | count" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml new file mode 100644 index 000000000..0849f2a2e --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/glusterfs_registry.yml @@ -0,0 +1,75 @@ +--- +- set_fact: + glusterfs_timeout: "{{ openshift_storage_glusterfs_registry_timeout }}" + glusterfs_namespace: "{{ openshift_storage_glusterfs_registry_namespace }}" + glusterfs_is_native: "{{ openshift_storage_glusterfs_registry_is_native }}" + glusterfs_name: "{{ openshift_storage_glusterfs_registry_name }}" + glusterfs_nodeselector: "{{ openshift_storage_glusterfs_registry_nodeselector | default(['storagenode', openshift_storage_glusterfs_registry_name] | join('=')) | map_from_pairs }}" + glusterfs_storageclass: "{{ openshift_storage_glusterfs_registry_storageclass }}" + glusterfs_image: "{{ openshift_storage_glusterfs_registry_image }}" + glusterfs_version: "{{ openshift_storage_glusterfs_registry_version }}" + glusterfs_wipe: "{{ openshift_storage_glusterfs_registry_wipe }}" + glusterfs_heketi_is_native: "{{ openshift_storage_glusterfs_registry_heketi_is_native }}" + glusterfs_heketi_is_missing: "{{ openshift_storage_glusterfs_registry_heketi_is_missing }}" + glusterfs_heketi_deploy_is_missing: "{{ openshift_storage_glusterfs_registry_heketi_deploy_is_missing }}" + glusterfs_heketi_image: "{{ openshift_storage_glusterfs_registry_heketi_image }}" + glusterfs_heketi_version: "{{ openshift_storage_glusterfs_registry_heketi_version }}" + glusterfs_heketi_admin_key: "{{ openshift_storage_glusterfs_registry_heketi_admin_key }}" + glusterfs_heketi_user_key: "{{ openshift_storage_glusterfs_registry_heketi_user_key }}" + glusterfs_heketi_topology_load: "{{ openshift_storage_glusterfs_registry_heketi_topology_load }}" + glusterfs_heketi_wipe: "{{ openshift_storage_glusterfs_registry_heketi_wipe }}" + glusterfs_heketi_url: "{{ openshift_storage_glusterfs_registry_heketi_url }}" + glusterfs_heketi_port: "{{ openshift_storage_glusterfs_registry_heketi_port }}" + glusterfs_nodes: "{{ groups.glusterfs_registry | default(groups.glusterfs) }}" + +- include: glusterfs_common.yml + when: + - glusterfs_nodes | default([]) | count > 0 + - "'glusterfs' not in groups or glusterfs_nodes != groups.glusterfs" + +- name: Delete pre-existing GlusterFS registry resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name }}" + state: absent + with_items: + - kind: "svc" + name: "glusterfs-{{ glusterfs_name }}-endpoints" + failed_when: False + +- name: Generate GlusterFS registry endpoints + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-endpoints.yml.j2" + dest: "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml" + +- name: Copy GlusterFS registry service + template: + src: "{{ openshift.common.examples_content_version }}/glusterfs-registry-service.yml.j2" + dest: "{{ mktemp.stdout }}/glusterfs-registry-service.yml" + +- name: Create GlusterFS registry endpoints + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: endpoints + name: "glusterfs-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/glusterfs-registry-endpoints.yml" + +- name: Create GlusterFS registry service + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: service + name: "glusterfs-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/glusterfs-registry-service.yml" + +- name: Check if GlusterFS registry volume exists + command: "{{ glusterfs_heketi_client }} volume list" + register: registry_volume + +- name: Create GlusterFS registry volume + command: "{{ glusterfs_heketi_client }} volume create --size={{ openshift.hosted.registry.storage.volume.size | replace('Gi','') }} --name={{ openshift.hosted.registry.storage.glusterfs.path }}" + when: "openshift.hosted.registry.storage.glusterfs.path not in registry_volume.stdout" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml new file mode 100644 index 000000000..ea9b1fe1f --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part1.yml @@ -0,0 +1,55 @@ +--- +- name: Copy initial heketi resource files + copy: + src: "{{ openshift.common.examples_content_version }}/{{ item }}" + dest: "{{ mktemp.stdout }}/{{ item }}" + with_items: + - "deploy-heketi-template.yml" + +- name: Create heketi topology secret + oc_secret: + namespace: "{{ glusterfs_namespace }}" + state: present + name: "heketi-{{ glusterfs_name }}-topology-secret" + force: True + files: + - name: topology.json + path: "{{ mktemp.stdout }}/topology.json" + +- name: Create deploy-heketi template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: "deploy-heketi" + state: present + files: + - "{{ mktemp.stdout }}/deploy-heketi-template.yml" + +- name: Deploy deploy-heketi pod + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "deploy-heketi" + create: True + params: + IMAGE_NAME: "{{ glusterfs_heketi_image }}" + IMAGE_VERSION: "{{ glusterfs_heketi_version }}" + HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}" + HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" + HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" + HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + TOPOLOGY_PATH: "{{ mktemp.stdout }}" + +- name: Wait for deploy-heketi pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=deploy-heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + until: + - "heketi_pod.results.results[0]['items'] | count > 0" + # Pod's 'Ready' status must be True + - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" diff --git a/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml new file mode 100644 index 000000000..63009c539 --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/heketi_deploy_part2.yml @@ -0,0 +1,132 @@ +--- +- name: Create heketi DB volume + command: "{{ glusterfs_heketi_client }} setup-openshift-heketi-storage --listfile /tmp/heketi-storage.json" + register: setup_storage + +- name: Copy heketi-storage list + shell: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} cat /tmp/heketi-storage.json > {{ mktemp.stdout }}/heketi-storage.json" + +# This is used in the subsequent task +- name: Copy the admin client config + command: > + cp {{ openshift.common.config_base }}/master/admin.kubeconfig {{ mktemp.stdout }}/admin.kubeconfig + changed_when: False + check_mode: no + +# Need `command` here because heketi-storage.json contains multiple objects. +- name: Copy heketi DB to GlusterFS volume + command: "{{ openshift.common.client_binary }} --config={{ mktemp.stdout }}/admin.kubeconfig create -f {{ mktemp.stdout }}/heketi-storage.json -n {{ glusterfs_namespace }}" + when: setup_storage.rc == 0 + +- name: Wait for copy job to finish + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: job + state: list + name: "heketi-storage-copy-job" + register: heketi_job + until: + - "'results' in heketi_job.results and heketi_job.results.results | count > 0" + # Pod's 'Complete' status must be True + - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Complete'}) | map('bool') | select | list | count == 1" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + failed_when: + - "'results' in heketi_job.results" + - "heketi_job.results.results | count > 0" + # Fail when pod's 'Failed' status is True + - "heketi_job.results.results | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Failed'}) | map('bool') | select | list | count == 1" + when: setup_storage.rc == 0 + +- name: Delete deploy resources + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: "{{ item.kind }}" + name: "{{ item.name | default(omit) }}" + selector: "{{ item.selector | default(omit) }}" + state: absent + with_items: + - kind: "template,route,service,jobs,dc,secret" + selector: "deploy-heketi" + - kind: "svc" + name: "heketi-storage-endpoints" + - kind: "secret" + name: "heketi-{{ glusterfs_name }}-topology-secret" + +- name: Generate heketi endpoints + template: + src: "{{ openshift.common.examples_content_version }}/heketi-endpoints.yml.j2" + dest: "{{ mktemp.stdout }}/heketi-endpoints.yml" + +- name: Generate heketi service + template: + src: "{{ openshift.common.examples_content_version }}/heketi-service.yml.j2" + dest: "{{ mktemp.stdout }}/heketi-service.yml" + +- name: Create heketi endpoints + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: endpoints + name: "heketi-db-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/heketi-endpoints.yml" + +- name: Create heketi service + oc_obj: + namespace: "{{ glusterfs_namespace }}" + state: present + kind: service + name: "heketi-db-{{ glusterfs_name }}-endpoints" + files: + - "{{ mktemp.stdout }}/heketi-service.yml" + +- name: Copy heketi template + copy: + src: "{{ openshift.common.examples_content_version }}/heketi-template.yml" + dest: "{{ mktemp.stdout }}/heketi-template.yml" + +- name: Create heketi template + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: template + name: heketi + state: present + files: + - "{{ mktemp.stdout }}/heketi-template.yml" + +- name: Deploy heketi pod + oc_process: + namespace: "{{ glusterfs_namespace }}" + template_name: "heketi" + create: True + params: + IMAGE_NAME: "{{ glusterfs_heketi_image }}" + IMAGE_VERSION: "{{ glusterfs_heketi_version }}" + HEKETI_ROUTE: "{{ glusterfs_heketi_url | default(['heketi-',glusterfs_name]|join) }}" + HEKETI_USER_KEY: "{{ glusterfs_heketi_user_key }}" + HEKETI_ADMIN_KEY: "{{ glusterfs_heketi_admin_key }}" + HEKETI_KUBE_NAMESPACE: "{{ glusterfs_namespace }}" + CLUSTER_NAME: "{{ glusterfs_name }}" + +- name: Wait for heketi pod + oc_obj: + namespace: "{{ glusterfs_namespace }}" + kind: pod + state: list + selector: "glusterfs=heketi-{{ glusterfs_name }}-pod" + register: heketi_pod + until: + - "heketi_pod.results.results[0]['items'] | count > 0" + # Pod's 'Ready' status must be True + - "heketi_pod.results.results[0]['items'] | oo_collect(attribute='status.conditions') | oo_collect(attribute='status', filters={'type': 'Ready'}) | map('bool') | select | list | count == 1" + delay: 10 + retries: "{{ (glusterfs_timeout | int / 10) | int }}" + +- name: Set heketi-cli command + set_fact: + glusterfs_heketi_client: "{{ openshift.common.client_binary }} rsh --namespace={{ glusterfs_namespace }} {{ heketi_pod.results.results[0]['items'][0]['metadata']['name'] }} heketi-cli -s http://localhost:8080 --user admin --secret '{{ glusterfs_heketi_admin_key }}'" + +- name: Verify heketi service + command: "{{ glusterfs_heketi_client }} cluster list" + changed_when: False diff --git a/roles/openshift_storage_glusterfs/tasks/main.yml b/roles/openshift_storage_glusterfs/tasks/main.yml new file mode 100644 index 000000000..d2d8c6c10 --- /dev/null +++ b/roles/openshift_storage_glusterfs/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Create temp directory for doing work in + command: mktemp -d /tmp/openshift-glusterfs-ansible-XXXXXX + register: mktemp + changed_when: False + check_mode: no + +- include: glusterfs_config.yml + when: + - groups.glusterfs | default([]) | count > 0 + +- include: glusterfs_registry.yml + when: + - "groups.glusterfs_registry | default([]) | count > 0 or openshift.hosted.registry.storage.kind == 'glusterfs' or openshift.hosted.registry.storage.glusterfs.swap" + +- name: Delete temp directory + file: + name: "{{ mktemp.stdout }}" + state: absent + changed_when: False + check_mode: no diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 new file mode 100644 index 000000000..11c9195bb --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-endpoints.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: glusterfs-{{ glusterfs_name }}-endpoints +subsets: +- addresses: +{% for node in glusterfs_nodes %} + - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }} +{% endfor %} + ports: + - port: 1 diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 new file mode 100644 index 000000000..3f869d2b7 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-registry-service.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: glusterfs-{{ glusterfs_name }}-endpoints +spec: + ports: + - port: 1 +status: + loadBalancer: {} diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 new file mode 100644 index 000000000..2ec9a9e9a --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/glusterfs-storageclass.yml.j2 @@ -0,0 +1,11 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: glusterfs-{{ glusterfs_name }} +provisioner: kubernetes.io/glusterfs +parameters: + resturl: "http://{% if glusterfs_heketi_is_native %}{{ glusterfs_heketi_route }}{% else %}{{ glusterfs_heketi_url }}:{{ glusterfs_heketi_port }}{% endif %}" + restuser: "admin" + secretNamespace: "{{ glusterfs_namespace }}" + secretName: "heketi-{{ glusterfs_name }}-admin-secret" diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 new file mode 100644 index 000000000..99cbdf748 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-endpoints.yml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: heketi-db-{{ glusterfs_name }}-endpoints +subsets: +- addresses: +{% for node in glusterfs_nodes %} + - ip: {{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }} +{% endfor %} + ports: + - port: 1 diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 new file mode 100644 index 000000000..dcb896441 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/heketi-service.yml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: heketi-db-{{ glusterfs_name }}-endpoints +spec: + ports: + - port: 1 +status: + loadBalancer: {} diff --git a/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2 b/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2 new file mode 100644 index 000000000..33d8f9b36 --- /dev/null +++ b/roles/openshift_storage_glusterfs/templates/v3.6/topology.json.j2 @@ -0,0 +1,39 @@ +{ + "clusters": [ +{%- set clusters = {} -%} +{%- for node in glusterfs_nodes -%} + {%- set cluster = hostvars[node].glusterfs_cluster if 'glusterfs_cluster' in node else '1' -%} + {%- if cluster in clusters -%} + {%- set _dummy = clusters[cluster].append(node) -%} + {%- else -%} + {%- set _dummy = clusters.update({cluster: [ node, ]}) -%} + {%- endif -%} +{%- endfor -%} +{%- for cluster in clusters -%} + { + "nodes": [ +{%- for node in clusters[cluster] -%} + { + "node": { + "hostnames": { + "manage": [ + "{{ hostvars[node].glusterfs_hostname | default(hostvars[node].openshift.common.hostname) }}" + ], + "storage": [ + "{{ hostvars[node].glusterfs_ip | default(hostvars[node].openshift.common.ip) }}" + ] + }, + "zone": {{ hostvars[node].glusterfs_zone | default(1) }} + }, + "devices": [ +{%- for device in hostvars[node].glusterfs_devices -%} + "{{ device }}"{% if not loop.last %},{% endif %} +{%- endfor -%} + ] + }{% if not loop.last %},{% endif %} +{%- endfor -%} + ] + }{% if not loop.last %},{% endif %} +{%- endfor -%} + ] +} |