summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--inventory/byo/hosts.aep.example27
-rw-r--r--inventory/byo/hosts.origin.example26
-rw-r--r--inventory/byo/hosts.ose.example25
-rw-r--r--roles/docker/tasks/main.yml15
-rw-r--r--roles/openshift_builddefaults/meta/main.yml15
-rw-r--r--roles/openshift_builddefaults/tasks/main.yml24
-rw-r--r--roles/openshift_builddefaults/vars/main.yml15
-rw-r--r--roles/openshift_docker_facts/tasks/main.yml3
-rwxr-xr-xroles/openshift_facts/library/openshift_facts.py73
-rw-r--r--roles/openshift_facts/tasks/main.yml15
-rw-r--r--roles/openshift_master/meta/main.yml1
-rw-r--r--roles/openshift_master/templates/atomic-openshift-master.j216
-rw-r--r--roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j215
-rw-r--r--roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j215
-rw-r--r--roles/openshift_master_facts/vars/main.yml14
15 files changed, 283 insertions, 16 deletions
diff --git a/inventory/byo/hosts.aep.example b/inventory/byo/hosts.aep.example
index 30d31bc12..d57cb4947 100644
--- a/inventory/byo/hosts.aep.example
+++ b/inventory/byo/hosts.aep.example
@@ -342,6 +342,33 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
# be used with 1.0 and 3.0.
# openshift_use_dnsmasq=False
+# Global Proxy Configuration
+# These options configure HTTP_PROXY, HTTPS_PROXY, and NOPROXY environment
+# variables for docker and master services.
+#openshift_http_proxy=http://USER:PASSWORD@IPADDR:PORT
+#openshift_https_proxy=https://USER:PASSWORD@IPADDR:PORT
+#openshift_no_proxy='.hosts.example.com,some-host.com'
+#
+# Most environments don't require a proxy between openshift masters, nodes, and
+# etcd hosts. So automatically add those hostnames to the openshift_no_proxy list.
+# If all of your hosts share a common domain you may wish to disable this and
+# specify that domain above.
+#openshift_generate_no_proxy_hosts: True
+#
+# These options configure the BuildDefaults admission controller which injects
+# environment variables into Builds. These values will default to their
+# corresponding values above but you may set them independently. See BuildDefaults
+# documentation at https://docs.openshift.org/latest/admin_guide/build_defaults_overrides.html
+#openshift_builddefaults_http_proxy=openshift_http_proxy
+#openshift_builddefaults_https_proxy=openshift_https_proxy
+#openshift_builddefaults_no_proxy=openshift_noproxy
+#openshift_builddefaults_git_http_proxy=openshift_builddefaults_http_proxy
+#openshift_builddefaults_git_https_proxy=openshift_builddefaults_https_proxy
+# Or you may optionally define your own serialized as json
+#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}],"gitHTTPProxy":"http://proxy.example.com:3128","kind":"BuildDefaultsConfig"}}}'
+
+
+
# host group for masters
[masters]
aep3-master[1:3]-ansible.test.example.com
diff --git a/inventory/byo/hosts.origin.example b/inventory/byo/hosts.origin.example
index 1dc5abfcf..2a4b00964 100644
--- a/inventory/byo/hosts.origin.example
+++ b/inventory/byo/hosts.origin.example
@@ -347,6 +347,32 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
# be used with 1.0 and 3.0.
# openshift_use_dnsmasq=False
+# Global Proxy Configuration
+# These options configure HTTP_PROXY, HTTPS_PROXY, and NOPROXY environment
+# variables for docker and master services.
+#openshift_http_proxy=http://USER:PASSWORD@IPADDR:PORT
+#openshift_https_proxy=https://USER:PASSWORD@IPADDR:PORT
+#openshift_no_proxy='.hosts.example.com,some-host.com'
+#
+# Most environments don't require a proxy between openshift masters, nodes, and
+# etcd hosts. So automatically add those hostnames to the openshift_no_proxy list.
+# If all of your hosts share a common domain you may wish to disable this and
+# specify that domain above.
+#openshift_generate_no_proxy_hosts: True
+#
+# These options configure the BuildDefaults admission controller which injects
+# environment variables into Builds. These values will default to their
+# corresponding values above but you may set them independently. See BuildDefaults
+# documentation at https://docs.openshift.org/latest/admin_guide/build_defaults_overrides.html
+#openshift_builddefaults_http_proxy=openshift_http_proxy
+#openshift_builddefaults_https_proxy=openshift_https_proxy
+#openshift_builddefaults_no_proxy=openshift_noproxy
+#openshift_builddefaults_git_http_proxy=openshift_builddefaults_http_proxy
+#openshift_builddefaults_git_https_proxy=openshift_builddefaults_https_proxy
+# Or you may optionally define your own serialized as json
+#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}],"gitHTTPProxy":"http://proxy.example.com:3128","kind":"BuildDefaultsConfig"}}}'
+
+
# host group for masters
[masters]
ose3-master[1:3]-ansible.test.example.com
diff --git a/inventory/byo/hosts.ose.example b/inventory/byo/hosts.ose.example
index f3ce95f0c..2df6bd5e2 100644
--- a/inventory/byo/hosts.ose.example
+++ b/inventory/byo/hosts.ose.example
@@ -343,6 +343,31 @@ openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true',
# be used with 1.0 and 3.0.
# openshift_use_dnsmasq=False
+# Global Proxy Configuration
+# These options configure HTTP_PROXY, HTTPS_PROXY, and NOPROXY environment
+# variables for docker and master services.
+#openshift_http_proxy=http://USER:PASSWORD@IPADDR:PORT
+#openshift_https_proxy=https://USER:PASSWORD@IPADDR:PORT
+#openshift_no_proxy='.hosts.example.com,some-host.com'
+#
+# Most environments don't require a proxy between openshift masters, nodes, and
+# etcd hosts. So automatically add those hostnames to the openshift_no_proxy list.
+# If all of your hosts share a common domain you may wish to disable this and
+# specify that domain above.
+#openshift_generate_no_proxy_hosts: True
+#
+# These options configure the BuildDefaults admission controller which injects
+# environment variables into Builds. These values will default to their
+# corresponding values above but you may set them independently. See BuildDefaults
+# documentation at https://docs.openshift.org/latest/admin_guide/build_defaults_overrides.html
+#openshift_builddefaults_http_proxy=openshift_http_proxy
+#openshift_builddefaults_https_proxy=openshift_https_proxy
+#openshift_builddefaults_no_proxy=openshift_noproxy
+#openshift_builddefaults_git_http_proxy=openshift_builddefaults_http_proxy
+#openshift_builddefaults_git_https_proxy=openshift_builddefaults_https_proxy
+# Or you may optionally define your own serialized as json
+#openshift_builddefaults_json='{"BuildDefaults":{"configuration":{"apiVersion":"v1","env":[{"name":"HTTP_PROXY","value":"http://proxy.example.com.redhat.com:3128"},{"name":"NO_PROXY","value":"ose3-master.example.com"}],"gitHTTPProxy":"http://proxy.example.com:3128","kind":"BuildDefaultsConfig"}}}'
+
# host group for masters
[masters]
ose3-master[1:3]-ansible.test.example.com
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 4a47a60fa..0414ff21e 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -69,6 +69,21 @@
reg_flag: --insecure-registry
notify:
- restart docker
+
+- name: Set Proxy Settings
+ lineinfile:
+ dest: /etc/sysconfig/docker
+ regexp: '^{{ item.reg_conf_var }}=.*$'
+ line: "{{ item.reg_conf_var }}='{{ item.reg_fact_val }}'"
+ with_items:
+ - reg_conf_var: HTTP_PROXY
+ reg_fact_val: "{{ docker_http_proxy | default('') }}"
+ - reg_conf_var: HTTPS_PROXY
+ reg_fact_val: "{{ docker_https_proxy | default('') }}"
+ - reg_conf_var: NO_PROXY
+ reg_fact_val: "{{ docker_no_proxy | default('') | join(',') }}"
+ notify:
+ - restart docker
- name: Set various docker options
lineinfile:
diff --git a/roles/openshift_builddefaults/meta/main.yml b/roles/openshift_builddefaults/meta/main.yml
new file mode 100644
index 000000000..422d08400
--- /dev/null
+++ b/roles/openshift_builddefaults/meta/main.yml
@@ -0,0 +1,15 @@
+---
+galaxy_info:
+ author: Scott Dodson
+ description: OpenShift Build Defaults configuration
+ company: Red Hat, Inc.
+ license: Apache License, Version 2.0
+ min_ansible_version: 1.9
+ platforms:
+ - name: EL
+ versions:
+ - 7
+ categories:
+ - cloud
+dependencies:
+- role: openshift_facts
diff --git a/roles/openshift_builddefaults/tasks/main.yml b/roles/openshift_builddefaults/tasks/main.yml
new file mode 100644
index 000000000..c82aebe72
--- /dev/null
+++ b/roles/openshift_builddefaults/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Set builddefaults
+ openshift_facts:
+ role: builddefaults
+ # TODO: add ability to define builddefaults env vars sort of like this
+ # may need to move the config generation to a filter however.
+ # openshift_env: "{{ hostvars[inventory_hostname]
+ # | oo_merge_dicts(hostvars)
+ # | oo_openshift_env }}"
+ # openshift_env_structures:
+ # - 'openshift.builddefaults.env.*'
+ local_facts:
+ http_proxy: "{{ openshift_builddefaults_http_proxy | default(None) }}"
+ https_proxy: "{{ openshift_builddefaults_https_proxy | default(None) }}"
+ no_proxy: "{{ openshift_builddefaults_no_proxy | default(None) }}"
+ git_http_proxy: "{{ openshift_builddefaults_git_http_proxy | default(None) }}"
+ git_https_proxy: "{{ openshift_builddefaults_git_https_proxy | default(None) }}"
+
+- name: Set builddefaults config structure
+ openshift_facts:
+ role: builddefaults
+ local_facts:
+ config: "{{ openshift_builddefaults_json | default(builddefaults_yaml) }}"
+
diff --git a/roles/openshift_builddefaults/vars/main.yml b/roles/openshift_builddefaults/vars/main.yml
new file mode 100644
index 000000000..9727c73a5
--- /dev/null
+++ b/roles/openshift_builddefaults/vars/main.yml
@@ -0,0 +1,15 @@
+---
+builddefaults_yaml:
+ BuildDefaults:
+ configuration:
+ apiVersion: v1
+ kind: BuildDefaultsConfig
+ gitHTTPProxy: "{{ openshift.builddefaults.git_http_proxy | default('', true) }}"
+ gitHTTPSProxy: "{{ openshift.builddefaults.git_https_proxy | default('', true) }}"
+ env:
+ - name: HTTP_PROXY
+ value: "{{ openshift.builddefaults.http_proxy | default('', true) }}"
+ - name: HTTPS_PROXY
+ value: "{{ openshift.builddefaults.https_proxy | default('', true) }}"
+ - name: NO_PROXY
+ value: "{{ openshift.builddefaults.no_proxy | default('', true) | join(',') }}"
diff --git a/roles/openshift_docker_facts/tasks/main.yml b/roles/openshift_docker_facts/tasks/main.yml
index 89393168b..3acd2bba8 100644
--- a/roles/openshift_docker_facts/tasks/main.yml
+++ b/roles/openshift_docker_facts/tasks/main.yml
@@ -27,6 +27,9 @@
docker_log_options: "{{ openshift.docker.log_options | default(omit) }}"
docker_push_dockerhub: "{{ openshift.docker.disable_push_dockerhub
| default(omit) }}"
+ docker_http_proxy: "{{ openshift.common.http_proxy | default(omit) }}"
+ docker_https_proxy: "{{ openshift.common.https_proxy | default(omit) }}"
+ docker_no_proxy: "{{ openshift.common.no_proxy | default(omit) }}"
- set_fact:
docker_options: >
diff --git a/roles/openshift_facts/library/openshift_facts.py b/roles/openshift_facts/library/openshift_facts.py
index 7de8b1006..f733fd5a8 100755
--- a/roles/openshift_facts/library/openshift_facts.py
+++ b/roles/openshift_facts/library/openshift_facts.py
@@ -1327,6 +1327,23 @@ def get_local_facts_from_file(filename):
return local_facts
+def sort_unique(alist):
+ """ Sorts and de-dupes a list
+
+ Args:
+ list: a list
+ Returns:
+ list: a sorted de-duped list
+ """
+
+ alist.sort()
+ out = list()
+ for i in alist:
+ if i not in out:
+ out.append(i)
+
+ return out
+
def safe_get_bool(fact):
""" Get a boolean fact safely.
@@ -1337,6 +1354,58 @@ def safe_get_bool(fact):
"""
return bool(strtobool(str(fact)))
+def set_proxy_facts(facts):
+ """ Set global proxy facts and promote defaults from http_proxy, https_proxy,
+ no_proxy to the more specific builddefaults and builddefaults_git vars.
+ 1. http_proxy, https_proxy, no_proxy
+ 2. builddefaults_*
+ 3. builddefaults_git_*
+
+ Args:
+ facts(dict): existing facts
+ Returns:
+ facts(dict): Updated facts with missing values
+ """
+ if 'common' in facts:
+ common = facts['common']
+ if 'http_proxy' in common or 'https_proxy' in common:
+ if 'generate_no_proxy_hosts' in common and \
+ common['generate_no_proxy_hosts']:
+ if 'no_proxy' in common and \
+ isinstance(common['no_proxy'], basestring):
+ common['no_proxy'] = common['no_proxy'].split(",")
+ else:
+ common['no_proxy'] = []
+ if 'no_proxy_internal_hostnames' in common:
+ common['no_proxy'].extend(common['no_proxy_internal_hostnames'].split(','))
+ common['no_proxy'].append('.' + common['dns_domain'])
+ common['no_proxy'].append(common['hostname'])
+ common['no_proxy'] = sort_unique(common['no_proxy'])
+ facts['common'] = common
+
+ if 'builddefaults' in facts:
+ facts['master']['admission_plugin_config'] = dict()
+ builddefaults = facts['builddefaults']
+ common = facts['common']
+ if 'http_proxy' not in builddefaults and 'http_proxy' in common:
+ builddefaults['http_proxy'] = common['http_proxy']
+ if 'https_proxy' not in builddefaults and 'https_proxy' in common:
+ builddefaults['https_proxy'] = common['https_proxy']
+ if 'no_proxy' not in builddefaults and 'no_proxy' in common:
+ builddefaults['no_proxy'] = common['no_proxy']
+ if 'git_http_proxy' not in builddefaults and 'http_proxy' in builddefaults:
+ builddefaults['git_http_proxy'] = builddefaults['http_proxy']
+ if 'git_https_proxy' not in builddefaults and 'https_proxy' in builddefaults:
+ builddefaults['git_https_proxy'] = builddefaults['https_proxy']
+ if 'admission_plugin_config' not in builddefaults:
+ builddefaults['admission_plugin_config'] = dict()
+ if 'config' in builddefaults and ('http_proxy' in builddefaults or \
+ 'https_proxy' in builddefaults):
+ facts['master']['admission_plugin_config'].update(builddefaults['config'])
+ facts['builddefaults'] = builddefaults
+
+ return facts
+
# pylint: disable=too-many-statements
def set_container_facts_if_unset(facts):
""" Set containerized facts.
@@ -1470,7 +1539,8 @@ class OpenShiftFacts(object):
Raises:
OpenShiftFactsUnsupportedRoleError:
"""
- known_roles = ['cloudprovider',
+ known_roles = ['builddefaults',
+ 'cloudprovider',
'common',
'docker',
'etcd',
@@ -1558,6 +1628,7 @@ class OpenShiftFacts(object):
facts = set_manageiq_facts_if_unset(facts)
facts = set_aggregate_facts(facts)
facts = set_etcd_facts_if_unset(facts)
+ facts = set_proxy_facts(facts)
if not safe_get_bool(facts['common']['is_containerized']):
facts = set_installed_variant_rpm_facts(facts)
return dict(openshift=facts)
diff --git a/roles/openshift_facts/tasks/main.yml b/roles/openshift_facts/tasks/main.yml
index 36def57c8..7510e4e39 100644
--- a/roles/openshift_facts/tasks/main.yml
+++ b/roles/openshift_facts/tasks/main.yml
@@ -33,3 +33,18 @@
is_containerized: "{{ l_is_containerized | default(None) }}"
public_hostname: "{{ openshift_public_hostname | default(None) }}"
public_ip: "{{ openshift_public_ip | default(None) }}"
+
+# had to be done outside of the above because hostname isn't yet set
+- name: Gather hostnames for proxy configuration
+ openshift_facts:
+ role: common
+ local_facts:
+ http_proxy: "{{ openshift_http_proxy | default(None) }}"
+ https_proxy: "{{ openshift_https_proxy | default(None) }}"
+ no_proxy: "{{ openshift_no_proxy | default(None) }}"
+ generate_no_proxy_hosts: "{{ openshift_generate_no_proxy_hosts | default(True) }}"
+ no_proxy_internal_hostnames: "{{ hostvars | oo_select_keys(groups['oo_nodes_to_config']
+ | union(groups['oo_masters_to_config'])
+ | union(groups['oo_etcd_to_config'] | default([])))
+ | oo_collect('openshift.common.hostname') | default([]) | join (',')
+ }}"
diff --git a/roles/openshift_master/meta/main.yml b/roles/openshift_master/meta/main.yml
index 0d4241e2c..e882e0b8b 100644
--- a/roles/openshift_master/meta/main.yml
+++ b/roles/openshift_master/meta/main.yml
@@ -15,5 +15,6 @@ dependencies:
- role: openshift_docker
- role: openshift_cli
- role: openshift_cloud_provider
+- role: openshift_builddefaults
- role: openshift_master_facts
- role: openshift_hosted_facts
diff --git a/roles/openshift_master/templates/atomic-openshift-master.j2 b/roles/openshift_master/templates/atomic-openshift-master.j2
index 862cfa8f1..c70f3ec57 100644
--- a/roles/openshift_master/templates/atomic-openshift-master.j2
+++ b/roles/openshift_master/templates/atomic-openshift-master.j2
@@ -10,8 +10,14 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
{% endif %}
# Proxy configuration
-# Origin uses standard HTTP_PROXY environment variables. Be sure to set
-# NO_PROXY for your master
-#NO_PROXY=master.example.com
-#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
-#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
+# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
+{% if 'http_proxy' in openshift.common or 'https_proxy' in openshift.common %}
+HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.master.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+{% else %}
+#HTTP_PROXY=http://user:pass@proxy.example.com
+#HTTPS_PROXY=http://user:pass@proxy.example.com
+#NO_PROXY='.hosts.example.com'
+{% endif %}
+
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
index 69754ee10..549ebe5ab 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-api.j2
@@ -10,8 +10,13 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
{% endif %}
# Proxy configuration
-# Origin uses standard HTTP_PROXY environment variables. Be sure to set
-# NO_PROXY for your master
-#NO_PROXY=master.example.com
-#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
-#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
+# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
+{% if 'http_proxy' or 'https_proxy' in openshift.common %}
+HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.master.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+{% else %}
+#HTTP_PROXY=http://user:pass@proxy.example.com
+#HTTPS_PROXY=http://user:pass@proxy.example.com
+#NO_PROXY='.hosts.example.com'
+{% endif %}
diff --git a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2 b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
index 048a4305a..08dc87d2e 100644
--- a/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
+++ b/roles/openshift_master/templates/native-cluster/atomic-openshift-master-controllers.j2
@@ -10,8 +10,13 @@ AWS_SECRET_ACCESS_KEY={{ openshift.cloudprovider.aws.secret_key }}
{% endif %}
# Proxy configuration
-# Origin uses standard HTTP_PROXY environment variables. Be sure to set
-# NO_PROXY for your master
-#NO_PROXY=master.example.com
-#HTTP_PROXY=http://USER:PASSWORD@IPADDR:PORT
-#HTTPS_PROXY=https://USER:PASSWORD@IPADDR:PORT
+# See https://docs.openshift.com/enterprise/latest/install_config/install/advanced_install.html#configuring-global-proxy
+{% if 'http_proxy' or 'https_proxy' in openshift.common %}
+HTTP_PROXY='{{ openshift.common.http_proxy | default('') }}'
+HTTPS_PROXY='{{ openshift.common.https_proxy | default('')}}'
+NO_PROXY='{{ openshift.common.no_proxy | default('') | join(',') }},{{ openshift.master.portal_net }},{{ openshift.master.sdn_cluster_network_cidr }}'
+{% else %}
+#HTTP_PROXY=http://user:pass@proxy.example.com
+#HTTPS_PROXY=http://user:pass@proxy.example.com
+#NO_PROXY='.hosts.example.com'
+{% endif %}
diff --git a/roles/openshift_master_facts/vars/main.yml b/roles/openshift_master_facts/vars/main.yml
new file mode 100644
index 000000000..3b0ee2761
--- /dev/null
+++ b/roles/openshift_master_facts/vars/main.yml
@@ -0,0 +1,14 @@
+builddefaults_yaml:
+ BuildDefaults:
+ configuration:
+ apiVersion: v1
+ kind: BuildDefaultsConfig
+ gitHTTPProxy: "{{ openshift.master.builddefaults_git_http_proxy | default(omit, true) }}"
+ gitHTTPSProxy: "{{ openshift.master.builddefaults_git_https_proxy | default(omit, true) }}"
+ env:
+ - name: HTTP_PROXY
+ value: "{{ openshift.master.builddefaults_http_proxy | default(omit, true) }}"
+ - name: HTTPS_PROXY
+ value: "{{ openshift.master.builddefaults_https_proxy | default(omit, true) }}"
+ - name: NO_PROXY
+ value: "{{ openshift.master.builddefaults_no_proxy | default(omit, true) | join(',') }}" \ No newline at end of file