diff --git a/playbook.yml b/playbook.yml index ec0ffab..eec99dd 100644 --- a/playbook.yml +++ b/playbook.yml @@ -11,16 +11,17 @@ # the connected AMD GPU is automatically discovered - assumes one # on swap to other AMD cards to avoid instability: # 'rm -rfv /etc/tuned/*amdgpu*' - gpu_clock_min: "2200" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile - gpu_clock_max: "2725" # default somewhere around 2529 to 2660 + gpu_clock_min: "750" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile + gpu_clock_max: "2675" # default somewhere around 2529 to 2660 gpumem_clock_static: "1075" gpu_power_multi: default: 0.869969040247678 # 281W - real default -# overclock: 0.928792569659443 # 300W - slight boost - overclock: 1.0 # 323W - full board capability + overclock: 0.928792569659443 # 300W - slight boost +# overclock: 1.0 # 323W - full board capability # optional, applies offset (+/-) to GPU voltage by provided mV # gpu_mv_offset: "-25" - gu_mv_offset: "+75" # add 50mV or 0.075V + # gpu_mv_offset: "+50" # add 50mV or 0.05V + gpu_mv_offset: "+25" # add 25mV or 0.025V # '-50' undervolts GPU core voltage 50mV or 0.05V # mostly untested, there be dragons/instability # diff --git a/roles/tuned_amdgpu/defaults/main.yml b/roles/tuned_amdgpu/defaults/main.yml index 14a9581..6792dcb 100644 --- a/roles/tuned_amdgpu/defaults/main.yml +++ b/roles/tuned_amdgpu/defaults/main.yml @@ -9,3 +9,4 @@ profile_name: "{{ item.0 }}" amdgpu_profiles: - default - overclock + - peak diff --git a/roles/tuned_amdgpu/files/profile-common.sh b/roles/tuned_amdgpu/files/profile-common.sh new file mode 100644 index 0000000..5970513 --- /dev/null +++ b/roles/tuned_amdgpu/files/profile-common.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# 'common' file sourced by other scripts under tuned profile +# +# dynamically determine the connected GPU using the DRM subsystem +CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) + +function get_hwmon_dir() { + CARD_DIR="/sys/class/drm/${1}/device/" + for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do + if [[ -f "${CANDIDATE}"/power1_cap ]]; then + # found a valid hwmon dir + echo "${CANDIDATE}" + fi + done +} + +# determine the hwmon directory +HWMON_DIR=$(get_hwmon_dir "${CARD}") + +# read all of the power profiles, used to get the IDs for assignment later +PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) + +# get power capability; later used determine limits +read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max + +# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl +# improves allocation latency +echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled + +# export determinations +export CARD +export HWMON_DIR +export PROFILE_MODES +export POWER_CAP diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml index a7bc037..4cc4e42 100644 --- a/roles/tuned_amdgpu/tasks/main.yml +++ b/roles/tuned_amdgpu/tasks/main.yml @@ -28,6 +28,14 @@ when: (fed_ppdtuned_swap is not defined) or ('tuned' not in ansible_facts.packages) become: true +- name: Ensure dynamic tuning is disabled + ansible.builtin.lineinfile: + path: /etc/tuned/tuned-main.conf + regexp: '^dynamic_tuning.*=' + line: 'dynamic_tuning = 0' + notify: Restart tuned + become: true + - name: Create custom profile directories ansible.builtin.file: state: directory @@ -38,20 +46,31 @@ - "{{ base_profiles }}" become: true -- name: Template AMDGPU control/reset scripts +- name: Copy 'common' AMDGPU script for all profiles + ansible.builtin.copy: + src: profile-common.sh + dest: "/etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-common.sh" + mode: "0644" # sourced, doesn't require executable bit + owner: root + group: root + notify: Restart tuned + with_nested: + - "{{ amdgpu_profiles }}" + - "{{ base_profiles }}" + become: true + +- name: Template custom AMDGPU profile scripts ansible.builtin.template: - src: templates/amdgpu-clock.sh.j2 + src: amdgpu-profile-{{ item.0 }}.sh.j2 dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-clock.sh owner: root group: root mode: "0755" - with_nested: - - "{{ amdgpu_profiles }}" - - "{{ base_profiles }}" + loop: "{{ amdgpu_profiles | product(base_profiles) | list }}" notify: Restart tuned become: true -- name: Template custom tuned profiles +- name: Template tuned.conf for custom profiles ansible.builtin.template: src: templates/tuned.conf.j2 dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/tuned.conf diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 new file mode 100644 index 0000000..4bd1282 --- /dev/null +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/bash +# script for tuned AMDGPU clock control +# configures GPU power/clock characteristics +# clocks/power in 3D are dynamic based on need/usage +# +# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default +# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR) +# +# rendered by Ansible with environment-appropriate values: +# card #, eg: card0 +# path to discovered sysfs device files (power/clock/voltage control) +# +# AMDGPU driver/sysfs references: +# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html +# https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh + +{# begin the templated script for 'default' profiles to reset state #} +# set control mode back to auto +# attempts to dynamically set optimal power profile for (load) conditions +echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + +# reset any existing profile clock changes +echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# adjust power limit using multiplier against board capability +POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" + +# extract the power-saving profile ID number +PROF_DEFAULT_NUM=$(/usr/bin/awk '$0 ~ /BOOTUP_DEFAULT.*:/ {print $1}' <<< "$PROFILE_MODES") + +# reset power/clock heuristics to power-saving +echo "${PROF_DEFAULT_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 new file mode 100644 index 0000000..1f0aa3a --- /dev/null +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 @@ -0,0 +1,58 @@ +#!/bin/bash +# script for tuned AMDGPU clock control +# configures GPU power/clock characteristics +# clocks/power in 3D are dynamic based on need/usage +# +# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default +# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR) +# +# rendered by Ansible with environment-appropriate values: +# card #, eg: card0 +# path to discovered sysfs device files (power/clock/voltage control) +# +# AMDGPU driver/sysfs references: +# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html +# https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh + +{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} +# set the minimum GPU clock - for best performance, this should be near the maximum +# RX6000 series power management *sucks* +echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# set the maximum GPU clock +echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# set the GPU *memory* clock +# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range +# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow +echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage +{% if gpu_mv_offset is defined %} + +# offset GPU voltage {{ gpu_mv_offset }}mV +echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage +{% endif %} + +# commit the changes +echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# force GPU core and memory into highest clocks (fix flickering and poor power management) +# set manual control mode +# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files +echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + +# adjust power limit using multiplier against board capability +POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" + +# avoid display flickering, force OC'd memory to highest clock +echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk + +# extract the VR power profile ID number +PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") + +# force 'overclocked' profile to 'VR' power/clock heuristics +# latency/frame timing seemed favorable with relatively-close minimum clocks +echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 similarity index 61% rename from roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 rename to roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 index cc2dd2a..14105a8 100644 --- a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 @@ -13,53 +13,10 @@ # AMDGPU driver/sysfs references: # https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html # https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh -{# done this way to avoid issues with the card number possibly shifting after playbook run #} -# dynamically determine the connected GPU using the DRM subsystem -CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) - -function get_hwmon_dir() { - CARD_DIR="/sys/class/drm/${1}/device/" - for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do - if [[ -f "${CANDIDATE}"/power1_cap ]]; then - # found a valid hwmon dir - echo "${CANDIDATE}" - fi - done -} - -# determine the hwmon directory -HWMON_DIR=$(get_hwmon_dir "${CARD}") - -# read all of the power profiles, used to get the IDs for assignment later -PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) - -# get power capability; later used determine limits -read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max - -# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl -# improves allocation latency -echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled - -{# begin the templated script for 'default' profiles to reset state #} -{% if 'default' in profile_name %} -# set control mode back to auto -# attempts to dynamically set optimal power profile for (load) conditions -echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# reset any existing profile clock changes -echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# adjust power limit using multiplier against board capability -POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') -echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" - -# extract the power-saving profile ID number -PROF_POWER_SAVING_NUM=$(/usr/bin/awk '$0 ~ /POWER_SAVING.*:/ {print $1}' <<< "$PROFILE_MODES") - -# reset power/clock heuristics to power-saving -echo "${PROF_POWER_SAVING_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode -{% else %} {# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} # set the minimum GPU clock - for best performance, this should be near the maximum # RX6000 series power management *sucks* @@ -107,4 +64,3 @@ echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mod # ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500 # followup: doesn't work that well in practice, still flaky on clocks/frame times #echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level -{% endif %}