diff --git a/README.md b/README.md index 24d0642..91f2b7f 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,36 @@ Two _'profiles'_ are in each name: | `desktop-amdgpu-overclock` | Includes the (assumed) existing `desktop` tuned profile.

Adjusts the GPU power limit, clocks, _and_ the voltage curve. | | `desktop-amdgpu-peak` | Includes the (assumed) existing `desktop` tuned profile.

Same as the `overclock` profile, but locks clocks to their highest configured values | -## Notable variables +## Config -These are the variables you're likely to want to change. They are defined in [playbook.yml](playbook.yml) +The playbook will render this config file: `/etc/tuned/amdgpu-profile-vars.conf` + +Here is a preview: + +```shell +gpu_clock_min=500 +gpu_clock_max=2715 +gpumem_clock_static=1075 +gpu_power_multi_def=0.869969040247678 +gpu_power_multi_oc=1.0 +gpu_mv_offset=+60 +``` + +Changes will require switching `tuned` profiles to re-read/apply the config. + +One can use [gamemode](https://wiki.archlinux.org/title/Gamemode) to do this automatically on start/stop. + +Sample `~/.config/gamemode.ini`: + +```ini +[custom] +start=tuned-adm profile latency-performance-amdgpu-overclock +end=tuned-adm profile latency-performance-amdgpu-default +``` + +## Variables + +These are the variables you'll want to change/consider. | Variable | Description | |------------------------|---------------------------------------------------------------------------------------| @@ -45,4 +72,5 @@ These are the variables you're likely to want to change. They are defined in [p | gpumem_clock_static | Sets the _static_ memory clock for the GPU (in `MHz`). This is *not* the _effective_ data rate. That is a multiple of this depending on the type of VRAM.

To avoid flickering this does *not* change dynamically with load. | | gpu_mv_offset | GPU core voltage offset. Takes +/- some integer in millivolts. Can be used to both over _and_ under volt. eg: `-50` _(undervolt `50mV` or `0.05V`)_ | | base_profiles | List of base tuned profiles to clone in the new AMDGPU profiles. Defaults based on `Fedora` | -| gpu_power_multi | Dictionary with two keys, `default` and `overclock`. Expects two floats to set a power limit relative to the board _capability_. Example: `1.0` is full board capability, `0.5` is 50%. | +| gpu_power_multi_def | Float between `0.0` and `1.0`; controls power limit relative to the board _capability_. For _'default'_-named power profiles. | +| gpu_power_multi_oc | Similar to `gpu_power_multi_def`, for _'overclock'_-named power profiles. | diff --git a/host_vars/localhost.yml b/host_vars/localhost.yml index fd9fc69..737568b 100644 --- a/host_vars/localhost.yml +++ b/host_vars/localhost.yml @@ -9,23 +9,21 @@ # default 500, for best performance: near maximum. applies with 'overclock' tuned profile gpu_clock_min: "500" -# gpu_clock_max: "2675" # default somewhere around 2529 to 2660. +# gpu_clock_max: "2675" # default 2524 for my card gpu_clock_max: "2715" gpumem_clock_static: "1075" -gpu_power_multi: - default: 0.869969040247678 # 281W - real default - # overclock: 0.928792569659443 # 300W - # overclock: 0.959752321981424 # 310W - overclock: 1.0 # 323W - full board capability -# optional, applies offset (+/-) to GPU voltage by provided mV -# only counts for peak and overlock profiles -# gpu_mv_offset: "-20" +gpu_power_multi_def: 0.869969040247678 # 281W - real default +gpu_power_multi_oc: 1.0 # full board power capability +# other multipliers for 323W boards like mine: +# 300W: 0.928792569659443 +# 310: 0.959752321981424 +# sample worksheet in 'power_max multi tab calculator.ods' +gpu_mv_offset: "+60" # add 60mV or 0.06V # gpu_mv_offset: "+150" # add 150mV or 0.15V # gpu_mv_offset: "+133" # add 133mV or 0.133V -gpu_mv_offset: "+60" # add 60mV or 0.06V # gpu_mv_offset: "+75" # add 75mV or 0.075V # gpu_mv_offset: "+125" # add 125mV or 0.125V # '-50' undervolts GPU core voltage 50mV or 0.05V; untested - here be dragons/instability diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml index 2855c51..9a42a78 100644 --- a/roles/tuned_amdgpu/tasks/main.yml +++ b/roles/tuned_amdgpu/tasks/main.yml @@ -40,6 +40,22 @@ notify: Restart tuned become: true +- name: Configure profile vars + ansible.builtin.lineinfile: + path: /etc/tuned/amdgpu-profile-vars.conf + line: "{{ item }}={{ vars[item] }}" + regexp: "^{{ item }}=" + create: true + mode: '0644' + when: vars[item] is defined + with_items: + - gpu_clock_min + - gpu_clock_max + - gpumem_clock_static + - gpu_power_multi_def + - gpu_power_multi_oc + - gpu_mv_offset + - name: Create custom profile directories ansible.builtin.file: state: directory @@ -50,27 +66,13 @@ - "{{ base_profiles }}" become: true -- name: Copy 'common' AMDGPU script for all profiles +- name: Template AMDGPU profile script ansible.builtin.template: - src: profile-common.sh.j2 - dest: "{{ (tuned_amdgpu_profile_dir, item.1 + '-amdgpu-' + item.0, 'amdgpu-common.sh') | ansible.builtin.path_join }}" - mode: "0644" # sourced, doesn't require executable bit - owner: root - group: root - notify: Restart tuned - with_nested: - - "{{ amdgpu_profiles }}" - - "{{ base_profiles }}" - become: true - -- name: Template custom AMDGPU profile scripts - ansible.builtin.template: - src: amdgpu-profile-{{ item.0 }}.sh.j2 - dest: "{{ (tuned_amdgpu_profile_dir, item.1 + '-amdgpu-' + item.0, 'amdgpu-clock.sh') | ansible.builtin.path_join }}" + src: amdgpu-profile.sh.j2 + dest: "{{ (tuned_amdgpu_profile_dir, 'amdgpu-clock.sh') | ansible.builtin.path_join }}" # place in base dir for all profiles, called w/ arg owner: root group: root mode: "0755" - loop: "{{ amdgpu_profiles | product(base_profiles) | list }}" notify: Restart tuned become: true diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 deleted file mode 100644 index 338c704..0000000 --- a/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -{{ ansible_managed | comment }} -# This is the 'default' script; resetting amdgpu clock/power/voltage tunables -# -# AMDGPU driver/sysfs references: -# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html -# https://docs.kernel.org/gpu/amdgpu/thermal.html -# -# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability -. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh - -{# begin the templated script for 'default' profiles to reset state #} -# set control mode back to auto -# attempts to dynamically set optimal power profile for (load) conditions -echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# reset any existing profile clock changes -echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# adjust power limit using multiplier against board capability -POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') -echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" - -# extract the power-saving profile ID number -PROF_DEFAULT_NUM=$(/usr/bin/awk '$0 ~ /BOOTUP_DEFAULT.*:/ {print $1}' <<< "$PROFILE_MODES") - -# reset power/clock heuristics to power-saving -echo "${PROF_DEFAULT_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 deleted file mode 100644 index 1a1982e..0000000 --- a/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -{{ ansible_managed | comment }} -# This is the 'overclock' script; applies amdgpu clock/power/voltage tunables -# -# AMDGPU driver/sysfs references: -# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html -# https://docs.kernel.org/gpu/amdgpu/thermal.html -# -# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability -. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh - -{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} -# set the minimum GPU clock - for best performance, this should be near the maximum -# RX6000 series power management *sucks* -echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# set the maximum GPU clock -echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# set the GPU *memory* clock -# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range -# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow -echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -{% if gpu_mv_offset is defined %} - -# offset GPU voltage {{ gpu_mv_offset }}mV -echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -{% endif %} - -# commit the changes -echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# force GPU core and memory into highest clocks (fix flickering and poor power management) -# set manual control mode -# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files -echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# adjust power limit using multiplier against board capability -POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}') -echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" - -# avoid display flickering, force OC'd memory to highest clock -echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk - -# extract the VR power profile ID number -PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") - -# force 'overclocked' profile to 'VR' power/clock heuristics -# latency/frame timing seemed favorable with relatively-close minimum clocks -echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 deleted file mode 100644 index 3165f7e..0000000 --- a/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -{{ ansible_managed | comment }} -# This is the 'peak' script; applies most-aggressive amdgpu clock/power/voltage tunables -# -# AMDGPU driver/sysfs references: -# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html -# https://docs.kernel.org/gpu/amdgpu/thermal.html -# -# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability -. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh - -{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} -# set the minimum GPU clock - for best performance, this should be near the maximum -# RX6000 series power management *sucks* -echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# set the maximum GPU clock -echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# set the GPU *memory* clock -# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range -# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow -echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -{% if gpu_mv_offset is defined %} - -# offset GPU voltage {{ gpu_mv_offset }}mV -echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -{% endif %} - -# commit the changes -echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# force GPU core and memory into highest clocks (fix flickering and poor power management) -# set manual control mode -# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files -echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# adjust power limit using multiplier against board capability -POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}') -echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" - -# pp_dpm_*clk settings are unintuitive, giving profiles that may be used -echo '1' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_sclk -echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk -echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_fclk -echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_socclk - -# extract the VR power profile ID number -PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") - -# force 'overclocked' profile to 'VR' power/clock heuristics -# latency/frame timing seemed favorable with relatively-close minimum clocks -echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode - -# note 4/8/2023: instead of 'manual'... try dealing with broken power management, force clocks to high -# ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500 -# followup: doesn't work that well in practice, still flaky on clocks/frame times -#echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile.sh.j2 new file mode 100644 index 0000000..57af6b6 --- /dev/null +++ b/roles/tuned_amdgpu/templates/amdgpu-profile.sh.j2 @@ -0,0 +1,130 @@ +#!/bin/bash +{{ ansible_managed | comment }} +# This is the script for 'amdgpu' profiles, (re)setting clock/power/voltage tunables +# +# AMDGPU driver/sysfs references: +# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html +# https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# Variables shown below named '$TUNED_...' are given values by '/etc/tuned/amdgpu-profile-vars.conf' +# +# determine the connected GPU using the DRM subsystem. FIXME: assumes one card, make configurable +CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) + +function get_hwmon_dir() { + CARD_DIR="/sys/class/drm/${1}/device/" + for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do + if [[ -f "${CANDIDATE}"/power1_cap ]]; then + # found a valid hwmon dir + echo "${CANDIDATE}" + fi + done +} + + +# determine the hwmon directory +HWMON_DIR=$(get_hwmon_dir "${CARD}") + +# read all of the power profiles, used to get the IDs for assignment later +PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) + +# get power capability; later used determine limits +read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max + +function amdgpu_profile_reset() { + + # set control mode back to auto + # attempts to dynamically set optimal power profile for (load) conditions + echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + + # reset any existing profile clock changes + echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + + # adjust power limit using multiplier against board capability + POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n="${TUNED_gpu_power_multi_def}" 'BEGIN {printf "%.0f", (m*n)}') + echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" + + # extract the power-saving profile ID number + PROF_DEFAULT_NUM=$(/usr/bin/awk '$0 ~ /BOOTUP_DEFAULT.*:/ {print $1}' <<< "$PROFILE_MODES") + + # reset power/clock heuristics to power-saving + echo "${PROF_DEFAULT_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode + + # delay before returning - have mercy, may be followed with other profile function calls + sleep 0.5 + +} + +function amdgpu_profile_overclock() { + echo "s 0 ${TUNED_gpu_clock_min}" | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + echo "s 1 ${TUNED_gpu_clock_max}" | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + echo "m 1 ${TUNED_gpumem_clock_static}" | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + + # under/over-voltage is considered optional or less likely to be defined, checked before use + if [[ -n ${TUNED_gpu_mv_offset} ]]; then + echo "vo ${TUNED_gpu_mv_offset}" | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + fi + + # commit the changes + echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + + # force GPU core and memory into highest clocks (fix flickering and poor power management) + # set manual control mode + # allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files + echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + + # adjust power limit using multiplier against board capability + POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n="${TUNED_gpu_power_multi_oc}" 'BEGIN {printf "%.0f", (m*n)}') + echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" + + # avoid display flickering, force OC'd memory to highest clock + echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk + + # extract the VR power profile ID number + PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") + + # force 'overclocked' profile to 'VR' power/clock heuristics + # latency/frame timing seemed favorable with relatively-close minimum clocks + echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode + + # delay before returning - have mercy, may be followed with other profile function calls + sleep 0.5 + +} + +function amdgpu_profile_help() { + echo "Usage: $0 {reset|overclock|peak}" + exit 1 +} + +# Check if an argument was provided +if [ -z "$1" ]; then + amdgpu_profile_help +fi + +# 'tuned' has trouble running scripts with args - treats '/path/to/script.sh arg1' as one complete item +# so, based on the active profile - run the right function +read -r -d '' TUNED_PROFILE < /etc/tuned/active_profile + +case "$TUNED_PROFILE" in + *-amdgpu-default) + amdgpu_profile_reset + ;; + *-amdgpu-overclock) + amdgpu_profile_reset + amdgpu_profile_overclock + ;; + *-amdgpu-peak) + # do everything the other profiles do... then set each clock table to their peak + amdgpu_profile_reset + amdgpu_profile_overclock + echo '1' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_sclk + echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk + echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_fclk + echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_socclk + # if I understand correctly, 'power_dpm_force_performance_level' at 'profile_peak' achieves similar... but precludes some control + ;; + *) + amdgpu_profile_help + ;; +esac diff --git a/roles/tuned_amdgpu/templates/profile-common.sh.j2 b/roles/tuned_amdgpu/templates/profile-common.sh.j2 deleted file mode 100644 index 3a6a2ef..0000000 --- a/roles/tuned_amdgpu/templates/profile-common.sh.j2 +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -{{ ansible_managed | comment }} -# 'common' file sourced by other scripts under tuned profile -# -# dynamically determine the connected GPU using the DRM subsystem -CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) - -function get_hwmon_dir() { - CARD_DIR="/sys/class/drm/${1}/device/" - for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do - if [[ -f "${CANDIDATE}"/power1_cap ]]; then - # found a valid hwmon dir - echo "${CANDIDATE}" - fi - done -} - -# determine the hwmon directory -HWMON_DIR=$(get_hwmon_dir "${CARD}") - -# read all of the power profiles, used to get the IDs for assignment later -PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) - -# get power capability; later used determine limits -read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max - -# export determinations -export CARD -export HWMON_DIR -export PROFILE_MODES -export POWER_CAP diff --git a/roles/tuned_amdgpu/templates/tuned.conf.j2 b/roles/tuned_amdgpu/templates/tuned.conf.j2 index 80c71e3..3ddf2f0 100644 --- a/roles/tuned_amdgpu/templates/tuned.conf.j2 +++ b/roles/tuned_amdgpu/templates/tuned.conf.j2 @@ -4,12 +4,16 @@ include={{ item.1 }} summary={{ item.1 }} + TCP/RAID tweaks + AMDGPU {{ item.0 }} +[variables] +include=/etc/tuned/amdgpu-profile-vars.conf + # reference/execute AMDGPU control script # used because some sysfs interfaces are transactional # cannot be set by a single param in 'tuned'/[sysfs] [gpuclockscript] type=script -script=${i:PROFILE_DIR}/amdgpu-clock.sh +script={{ (tuned_amdgpu_profile_dir, 'amdgpu-clock.sh') | ansible.builtin.path_join }} +{# call the state-managing script with the selected profile, item.0, as an argument #} {% if plugins is defined %} {% for section, options in plugins.items() %} {#+ give each plugin section some space +#}