Merge pull request #1003 from fraenki/haproxy_210_3

net/haproxy: 3rd feature set for release 2.10
This commit is contained in:
Frank Wall
2018-11-20 00:16:44 +01:00
committed by GitHub
8 changed files with 511 additions and 2 deletions
@@ -1104,6 +1104,136 @@ class SettingsController extends ApiControllerBase
);
}
/**
* retrieve cpu settings or return defaults
* @param $uuid item unique id
* @return array
*/
public function getCpuAction($uuid = null)
{
$mdlCP = new HAProxy();
if ($uuid != null) {
$node = $mdlCP->getNodeByReference('cpus.cpu.'.$uuid);
if ($node != null) {
// return node
return array("cpu" => $node->getNodes());
}
} else {
// generate new node, but don't save to disc
$node = $mdlCP->cpus->cpu->add();
return array("cpu" => $node->getNodes());
}
return array();
}
/**
* update cpu with given properties
* @param $uuid item unique id
* @return array
*/
public function setCpuAction($uuid)
{
if ($this->request->isPost() && $this->request->hasPost("cpu")) {
$mdlCP = new HAProxy();
if ($uuid != null) {
$node = $mdlCP->getNodeByReference('cpus.cpu.'.$uuid);
if ($node != null) {
$node->setNodes($this->request->getPost("cpu"));
return $this->save($mdlCP, $node, "cpu");
}
}
}
return array("result"=>"failed");
}
/**
* add new cpu and set with attributes from post
* @return array
*/
public function addCpuAction()
{
$result = array("result"=>"failed");
if ($this->request->isPost() && $this->request->hasPost("cpu")) {
$mdlCP = new HAProxy();
$node = $mdlCP->cpus->cpu->Add();
$node->setNodes($this->request->getPost("cpu"));
return $this->save($mdlCP, $node, "cpu");
}
return $result;
}
/**
* delete cpu by uuid
* @param $uuid item unique id
* @return array status
*/
public function delCpuAction($uuid)
{
$result = array("result"=>"failed");
if ($this->request->isPost()) {
$mdlCP = new HAProxy();
if ($uuid != null) {
if ($mdlCP->cpus->cpu->del($uuid)) {
// if item is removed, serialize to config and save
$mdlCP->serializeToConfig();
Config::getInstance()->save();
$result['result'] = 'deleted';
} else {
$result['result'] = 'not found';
}
}
}
return $result;
}
/**
* toggle cpu by uuid (enable/disable)
* @param $uuid item unique id
* @param $enabled desired state enabled(1)/disabled(0), leave empty for toggle
* @return array status
*/
public function toggleCpuAction($uuid, $enabled = null)
{
$result = array("result" => "failed");
if ($this->request->isPost()) {
$mdlCP = new HAProxy();
if ($uuid != null) {
$node = $mdlCP->getNodeByReference('cpus.cpu.' . $uuid);
if ($node != null) {
if ($enabled == "0" || $enabled == "1") {
$node->enabled = (string)$enabled;
} elseif ((string)$node->enabled == "1") {
$node->enabled = "0";
} else {
$node->enabled = "1";
}
$result['result'] = $node->enabled;
// if item has toggled, serialize to config and save
$mdlCP->serializeToConfig();
Config::getInstance()->save();
}
}
}
return $result;
}
/**
* search cpus
* @return array
*/
public function searchCpusAction()
{
$this->sessionClose();
$mdlCP = new HAProxy();
$grid = new UIModelGrid($mdlCP->cpus->cpu);
return $grid->fetchBindRequest(
$this->request,
array("enabled", "name", "process_id", "thread_id", "cpu_id"),
"name"
);
}
/**
* retrieve group settings or return defaults
* @param $uuid item unique id
@@ -56,6 +56,7 @@ class IndexController extends \OPNsense\Base\IndexController
$this->view->formDialogLua = $this->getForm("dialogLua");
$this->view->formDialogErrorfile = $this->getForm("dialogErrorfile");
$this->view->formDialogMapfile = $this->getForm("dialogMapfile");
$this->view->formDialogCpu = $this->getForm("dialogCpu");
// set additional view parameters
$mdlHAProxy = new \OPNsense\HAProxy\HAProxy();
$this->view->showIntro = (string)$mdlHAProxy->general->showIntro;
@@ -0,0 +1,33 @@
<form>
<field>
<id>cpu.enabled</id>
<label>Enabled</label>
<type>checkbox</type>
<help>Enable this CPU affinity rule.</help>
</field>
<field>
<id>cpu.name</id>
<label>Name</label>
<type>text</type>
<help>Choose a name for this CPU affinity rule.</help>
</field>
<field>
<id>cpu.process_id</id>
<label>Process ID</label>
<type>dropdown</type>
<help>Process ID that should bind to a specific CPU set. Any process IDs above nbproc are ignored.</help>
</field>
<field>
<id>cpu.thread_id</id>
<label>Thread ID</label>
<type>dropdown</type>
<help>Thread ID that should bind to a specific CPU set. Any thread IDs above nbthread are ignored.</help>
</field>
<field>
<id>cpu.cpu_id</id>
<label>CPU ID</label>
<type>select_multiple</type>
<allownew>true</allownew>
<help>Bind the process/thread ID to this CPU.</help>
</field>
</form>
@@ -217,6 +217,15 @@
<help><![CDATA[Set the maximum allowed time to wait for a new HTTP request to appear. Defaults to milliseconds. Optionally the unit may be specified as either "d", "h", "m", "s", "ms" or "us".]]></help>
<advanced>true</advanced>
</field>
<field>
<id>frontend.linkedCpuAffinityRules</id>
<label>CPU Affinity Rules</label>
<type>select_multiple</type>
<allownew>true</allownew>
<help><![CDATA[Choose CPU affinity rules that should be applied to this Public Service.]]></help>
<hint>Choose CPU affinity rules.</hint>
<advanced>true</advanced>
</field>
<field>
<label>Logging Options</label>
<type>header</type>
@@ -97,6 +97,12 @@
<help><![CDATA[Number of HAProxy processes to start.<br/><div class="text-info"><b>NOTE:</b> You may experience random issues in multi-process mode. For more information about the "nbproc" option please see the HAProxy Documentation.</div>]]></help>
<advanced>true</advanced>
</field>
<field>
<id>haproxy.general.tuning.nbthread</id>
<label>HAProxy threads</label>
<type>text</type>
<help><![CDATA[Number of threads to create for each HAProxy process.]]></help>
</field>
<field>
<id>haproxy.general.tuning.maxConnections</id>
<label>Maximum connections</label>
@@ -69,6 +69,13 @@
<ValidationMessage>Please specify a value between 1 and 128.</ValidationMessage>
<Required>Y</Required>
</nbproc>
<nbthread type="IntegerField">
<default>1</default>
<MinimumValue>1</MinimumValue>
<MaximumValue>1024</MaximumValue>
<ValidationMessage>Please specify a value between 1 and 1024.</ValidationMessage>
<Required>N</Required>
</nbthread>
<sslServerVerify type="OptionField">
<Required>Y</Required>
<default>ignore</default>
@@ -502,6 +509,18 @@
<ValidationMessage>Should be a number between 1 and 8 characters, optionally followed by either "d", "h", "m", "s", "ms" or "us".</ValidationMessage>
<Required>N</Required>
</tuning_timeoutHttpKeepAlive>
<linkedCpuAffinityRules type="ModelRelationField">
<Model>
<template>
<source>OPNsense.HAProxy.HAProxy</source>
<items>cpus.cpu</items>
<display>name</display>
</template>
</Model>
<ValidationMessage>Related CPU affinity rule not found</ValidationMessage>
<Multiple>Y</Multiple>
<Required>N</Required>
</linkedCpuAffinityRules>
<logging_dontLogNull type="BooleanField">
<default>0</default>
<Required>Y</Required>
@@ -2085,5 +2104,236 @@
</password>
</user>
</users>
<cpus>
<cpu type="ArrayField">
<id type="UniqueIdField">
<Required>Y</Required>
</id>
<enabled type="BooleanField">
<default>1</default>
<Required>Y</Required>
</enabled>
<name type="TextField">
<mask>/^[^\t^,^;^\.^\[^\]^\{^\}]{1,255}$/u</mask>
<ValidationMessage>Should be a string between 1 and 255 characters.</ValidationMessage>
<Required>Y</Required>
</name>
<process_id type="OptionField">
<Required>Y</Required>
<OptionValues>
<all>All HAProxy processes</all>
<odd>Processes with odd ID</odd>
<even>Processes with even ID</even>
<x1>Process 1</x1>
<x2>Process 2</x2>
<x3>Process 3</x3>
<x4>Process 4</x4>
<x5>Process 5</x5>
<x6>Process 6</x6>
<x7>Process 7</x7>
<x8>Process 8</x8>
<x9>Process 9</x9>
<x10>Process 10</x10>
<x11>Process 11</x11>
<x12>Process 12</x12>
<x13>Process 13</x13>
<x14>Process 14</x14>
<x15>Process 15</x15>
<x16>Process 16</x16>
<x17>Process 17</x17>
<x18>Process 18</x18>
<x19>Process 19</x19>
<x20>Process 20</x20>
<x21>Process 21</x21>
<x22>Process 22</x22>
<x23>Process 23</x23>
<x24>Process 24</x24>
<x25>Process 25</x25>
<x26>Process 26</x26>
<x27>Process 27</x27>
<x28>Process 28</x28>
<x29>Process 29</x29>
<x30>Process 30</x30>
<x31>Process 31</x31>
<x32>Process 32</x32>
<x33>Process 33</x33>
<x34>Process 34</x34>
<x35>Process 35</x35>
<x36>Process 36</x36>
<x37>Process 37</x37>
<x38>Process 38</x38>
<x39>Process 39</x39>
<x40>Process 40</x40>
<x41>Process 41</x41>
<x42>Process 42</x42>
<x43>Process 43</x43>
<x44>Process 44</x44>
<x45>Process 45</x45>
<x46>Process 46</x46>
<x47>Process 47</x47>
<x48>Process 48</x48>
<x49>Process 49</x49>
<x50>Process 50</x50>
<x51>Process 51</x51>
<x52>Process 52</x52>
<x53>Process 53</x53>
<x54>Process 54</x54>
<x55>Process 55</x55>
<x56>Process 56</x56>
<x57>Process 57</x57>
<x58>Process 58</x58>
<x59>Process 59</x59>
<x60>Process 60</x60>
<x61>Process 61</x61>
<x62>Process 62</x62>
<x63>Process 63</x63>
</OptionValues>
</process_id>
<thread_id type="OptionField">
<Required>Y</Required>
<OptionValues>
<all>All HAProxy threads </all>
<odd>Threads with odd ID</odd>
<even>Threads with even ID</even>
<x1>Thread 1</x1>
<x2>Thread 2</x2>
<x3>Thread 3</x3>
<x4>Thread 4</x4>
<x5>Thread 5</x5>
<x6>Thread 6</x6>
<x7>Thread 7</x7>
<x8>Thread 8</x8>
<x9>Thread 9</x9>
<x10>Thread 10</x10>
<x11>Thread 11</x11>
<x12>Thread 12</x12>
<x13>Thread 13</x13>
<x14>Thread 14</x14>
<x15>Thread 15</x15>
<x16>Thread 16</x16>
<x17>Thread 17</x17>
<x18>Thread 18</x18>
<x19>Thread 19</x19>
<x20>Thread 20</x20>
<x21>Thread 21</x21>
<x22>Thread 22</x22>
<x23>Thread 23</x23>
<x24>Thread 24</x24>
<x25>Thread 25</x25>
<x26>Thread 26</x26>
<x27>Thread 27</x27>
<x28>Thread 28</x28>
<x29>Thread 29</x29>
<x30>Thread 30</x30>
<x31>Thread 31</x31>
<x32>Thread 32</x32>
<x33>Thread 33</x33>
<x34>Thread 34</x34>
<x35>Thread 35</x35>
<x36>Thread 36</x36>
<x37>Thread 37</x37>
<x38>Thread 38</x38>
<x39>Thread 39</x39>
<x40>Thread 40</x40>
<x41>Thread 41</x41>
<x42>Thread 42</x42>
<x43>Thread 43</x43>
<x44>Thread 44</x44>
<x45>Thread 45</x45>
<x46>Thread 46</x46>
<x47>Thread 47</x47>
<x48>Thread 48</x48>
<x49>Thread 49</x49>
<x50>Thread 50</x50>
<x51>Thread 51</x51>
<x52>Thread 52</x52>
<x53>Thread 53</x53>
<x54>Thread 54</x54>
<x55>Thread 55</x55>
<x56>Thread 56</x56>
<x57>Thread 57</x57>
<x58>Thread 58</x58>
<x59>Thread 59</x59>
<x60>Thread 60</x60>
<x61>Thread 61</x61>
<x62>Thread 62</x62>
<x63>Thread 63</x63>
</OptionValues>
</thread_id>
<cpu_id type="OptionField">
<Required>Y</Required>
<Multiple>Y</Multiple>
<OptionValues>
<all>All CPUs</all>
<odd>CPUs with odd ID</odd>
<even>CPUs with even ID</even>
<x0>CPU 0</x0>
<x1>CPU 1</x1>
<x2>CPU 2</x2>
<x3>CPU 3</x3>
<x4>CPU 4</x4>
<x5>CPU 5</x5>
<x6>CPU 6</x6>
<x7>CPU 7</x7>
<x8>CPU 8</x8>
<x9>CPU 9</x9>
<x10>CPU 10</x10>
<x11>CPU 11</x11>
<x12>CPU 12</x12>
<x13>CPU 13</x13>
<x14>CPU 14</x14>
<x15>CPU 15</x15>
<x16>CPU 16</x16>
<x17>CPU 17</x17>
<x18>CPU 18</x18>
<x19>CPU 19</x19>
<x20>CPU 20</x20>
<x21>CPU 21</x21>
<x22>CPU 22</x22>
<x23>CPU 23</x23>
<x24>CPU 24</x24>
<x25>CPU 25</x25>
<x26>CPU 26</x26>
<x27>CPU 27</x27>
<x28>CPU 28</x28>
<x29>CPU 29</x29>
<x30>CPU 30</x30>
<x31>CPU 31</x31>
<x32>CPU 32</x32>
<x33>CPU 33</x33>
<x34>CPU 34</x34>
<x35>CPU 35</x35>
<x36>CPU 36</x36>
<x37>CPU 37</x37>
<x38>CPU 38</x38>
<x39>CPU 39</x39>
<x40>CPU 40</x40>
<x41>CPU 41</x41>
<x42>CPU 42</x42>
<x43>CPU 43</x43>
<x44>CPU 44</x44>
<x45>CPU 45</x45>
<x46>CPU 46</x46>
<x47>CPU 47</x47>
<x48>CPU 48</x48>
<x49>CPU 49</x49>
<x50>CPU 50</x50>
<x51>CPU 51</x51>
<x52>CPU 52</x52>
<x53>CPU 53</x53>
<x54>CPU 54</x54>
<x55>CPU 55</x55>
<x56>CPU 56</x56>
<x57>CPU 57</x57>
<x58>CPU 58</x58>
<x59>CPU 59</x59>
<x60>CPU 60</x60>
<x61>CPU 61</x61>
<x62>CPU 62</x62>
<x63>CPU 63</x63>
</OptionValues>
</cpu_id>
</cpu>
</cpus>
</items>
</model>
@@ -184,6 +184,19 @@ POSSIBILITY OF SUCH DAMAGE.
}
);
$("#grid-cpus").UIBootgrid(
{ search:'/api/haproxy/settings/searchCpus',
get:'/api/haproxy/settings/getCpu/',
set:'/api/haproxy/settings/setCpu/',
add:'/api/haproxy/settings/addCpu/',
del:'/api/haproxy/settings/delCpu/',
toggle:'/api/haproxy/settings/toggleCpu/',
options: {
rowCount:[10,25,50,100,500,1000]
}
}
);
// hook into on-show event for dialog to extend layout.
$('#DialogAcl').on('shown.bs.modal', function (e) {
$("#acl\\.expression").change(function(){
@@ -516,6 +529,7 @@ POSSIBILITY OF SUCH DAMAGE.
<li><a data-toggle="tab" id="errorfiles-tab" href="#errorfiles">{{ lang._('Error Messages') }}</a></li>
<li><a data-toggle="tab" href="#luas">{{ lang._('Lua Scripts') }}</a></li>
<li><a data-toggle="tab" href="#mapfiles">{{ lang._('Map Files') }}</a></li>
<li><a data-toggle="tab" href="#cpus">{{ lang._('CPU Affinity Rules') }}</a></li>
</ul>
</li>
</ul>
@@ -603,8 +617,9 @@ POSSIBILITY OF SUCH DAMAGE.
<li>{{ lang._("%sError Messages:%s Return a custom message instead of errors generated by HAProxy. Useful to overwrite HAProxy's internal error messages. The message must represent the full HTTP response and include required HTTP headers.") | format('<b>', '</b>') }}</li>
<li>{{ lang._("%sLua scripts:%s Include your own Lua code/scripts to extend HAProxy's functionality. The Lua code can be used in certain %sRules%s, for example.") | format('<b>', '</b>', '<b>', '</b>') }}</li>
<li>{{ lang._("%sMap Files:%s A map allows to map a data in input to an other one on output. For example, this makes it possible to map a large number of domains to backend pools without using the GUI. Map files need to be used in %sRules%s, otherwise they are ignored.") | format('<b>', '</b>', '<b>', '</b>') }}</li>
<li>{{ lang._("%sCPU Affinity Rules:%s This feature makes it possible to bind HAProxy's processes/threads to a specific CPU (or a CPU set). Furthermore it is possible to select CPU Affinity Rules in %sPublic Services%s to restrict them to a certain set of processes/threads/CPUs.") | format('<b>', '</b>', '<b>', '</b>') }}</li>
</ul>
<p>{{ lang._("For more details visit HAProxy's official documentation regarding the %sError Messages%s, %sLua Script%s and the %sMap Files%s features.") | format('<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-errorfile" target="_blank">', '</a>', '<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#lua-load" target="_blank">', '</a>', '<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#map" target="_blank">', '</a>') }}</p>
<p>{{ lang._("For more details visit HAProxy's official documentation regarding the %sError Messages%s, %sLua Script%s and the %sMap Files%s features. More information on HAProxy's CPU Affinity is also available %shere%s, %shere%s and %shere%s.") | format('<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-errorfile" target="_blank">', '</a>', '<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#lua-load" target="_blank">', '</a>', '<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#map" target="_blank">', '</a>' ,'<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#cpu-map" target="_blank">', '</a>' ,'<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#bind-process" target="_blank">', '</a>' ,'<a href="http://cbonte.github.io/haproxy-dconv/1.8/configuration.html#process" target="_blank">', '</a>') }}</p>
<br/>
</div>
</div>
@@ -1006,6 +1021,43 @@ POSSIBILITY OF SUCH DAMAGE.
<br/>
</div>
</div>
<div id="cpus" class="tab-pane fade">
<!-- tab page "cpus" -->
<table id="grid-cpus" class="table table-condensed table-hover table-striped table-responsive" data-editDialog="DialogCpu">
<thead>
<tr>
<th data-column-id="cpuid" data-type="number" data-visible="false">{{ lang._('CPU Rule ID') }}</th>
<th data-column-id="enabled" data-width="6em" data-type="string" data-formatter="rowtoggle">{{ lang._('Enabled') }}</th>
<th data-column-id="name" data-type="string">{{ lang._('Name') }}</th>
<th data-column-id="process_id" data-type="string">{{ lang._('Process ID') }}</th>
<th data-column-id="thread_id" data-type="string">{{ lang._('Thread ID') }}</th>
<th data-column-id="cpu_id" data-type="string">{{ lang._('CPU ID') }}</th>
<th data-column-id="commands" data-width="7em" data-formatter="commands" data-sortable="false">{{ lang._('Commands') }}</th>
<th data-column-id="uuid" data-type="string" data-identifier="true" data-visible="false">{{ lang._('ID') }}</th>
</tr>
</thead>
<tbody>
</tbody>
<tfoot>
<tr>
<td></td>
<td>
<button data-action="add" type="button" class="btn btn-xs btn-default"><span class="fa fa-plus"></span></button>
<button data-action="deleteSelected" type="button" class="btn btn-xs btn-default"><span class="fa fa-trash-o"></span></button>
</td>
</tr>
</tfoot>
</table>
<!-- apply button -->
<div class="col-md-12">
<hr/>
<button class="btn btn-primary" id="reconfigureAct-cpus" type="button"><b>{{ lang._('Apply') }}</b><i id="reconfigureAct_progress" class=""></i></button>
<button class="btn btn-primary" id="configtestAct-cpus" type="button"><b>{{ lang._('Test syntax') }}</b><i id="configtestAct_progress" class=""></i></button>
<br/>
<br/>
</div>
</div>
</div>
{# include dialogs #}
@@ -1020,3 +1072,4 @@ POSSIBILITY OF SUCH DAMAGE.
{{ partial("layout_partials/base_dialog",['fields':formDialogLua,'id':'DialogLua','label':lang._('Edit Lua Script')])}}
{{ partial("layout_partials/base_dialog",['fields':formDialogErrorfile,'id':'DialogErrorfile','label':lang._('Edit Error Message')])}}
{{ partial("layout_partials/base_dialog",['fields':formDialogMapfile,'id':'DialogMapfile','label':lang._('Edit Map File')])}}
{{ partial("layout_partials/base_dialog",['fields':formDialogCpu,'id':'DialogCpu','label':lang._('Edit CPU Affinity Rule')])}}
@@ -758,6 +758,16 @@ global
stats socket /var/run/haproxy.socket level admin
{% endif %}
nbproc {{OPNsense.HAProxy.general.tuning.nbproc}}
{% if OPNsense.HAProxy.general.tuning.nbthread|default('') != '' %}
nbthread {{OPNsense.HAProxy.general.tuning.nbthread}}
{% endif %}
{% if helpers.exists('OPNsense.HAProxy.cpus.cpu') %}
{% for cpu_map in helpers.toList('OPNsense.HAProxy.cpus.cpu') %}
{% if cpu_map.enabled == '1' %}
cpu-map {{cpu_map.process_id|replace('x', '')}}/{{cpu_map.thread_id|replace('x', '')}} {{cpu_map.cpu_id|replace('x', '')|replace(',', ' ')}}
{% endif %}
{% endfor %}
{% endif %}
{% if helpers.exists('OPNsense.HAProxy.general.tuning.maxConnections') %}
maxconn {{OPNsense.HAProxy.general.tuning.maxConnections}}
{% endif %}
@@ -969,10 +979,27 @@ frontend {{frontend.name}}
{% endif %}
{% endif %}
{% endif %}
{# # CPU affinity configuration #}
{% set bind_process = [] %}
{% set process_thread = [] %}
{% if frontend.linkedCpuAffinityRules|default('') != '' %}
{% for cpu_map in frontend.linkedCpuAffinityRules.split(',') %}
{% set cpu_map_data = helpers.getUUID(cpu_map) %}
{% if cpu_map_data.enabled == '1' %}
{# # Limit visibility to a certain set of processes #}
{% do bind_process.append(cpu_map_data.process_id|replace('x', '')) %}
{# # Restrict the list of processes/threads on which this listener is allowed to run #}
{% do process_thread.append('process ' ~ cpu_map_data.process_id|replace('x', '') ~ '/' ~ cpu_map_data.thread_id|replace('x', '')) %}
{% endif %}
{% endfor %}
{% if bind_process|length > 0 %}
bind-process {{bind_process|join(' ')}}
{% endif %}
{% endif %}
{# # bind/listen configuration #}
{% if frontend.bind|default("") != "" %}
{% for bind in frontend.bind.split(",") %}
bind {{bind}} name {{bind}} {% if frontend.bindOptions|default("") != "" %}{{ frontend.bindOptions }} {% endif %}{% if frontend.ssl_enabled == '1' and ssl_certs|default("") != "" %}ssl {{ ssl_options|join(' ') }} {{ ssl_certs|join(' ') }} {% endif %}
bind {{bind}} name {{bind}} {% if frontend.bindOptions|default("") != "" %}{{ frontend.bindOptions }} {% endif %}{% if frontend.ssl_enabled == '1' and ssl_certs|default("") != "" %}ssl {{ ssl_options|join(' ') }} {{ ssl_certs|join(' ') }} {% endif %}{% if process_thread|length > 0 %} {{ process_thread|join(' ') }} {% endif %}
{% endfor %}
{% endif %}