From ca723c732bc57ac996fd49e4460552e1a466d91d Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Wed, 3 Aug 2022 00:47:43 -0500 Subject: [PATCH] fix clock profile choice consistency when moving profiles --- playbook.yml | 53 ++++++++++++++++++++++++------------ templates/amdgpu-clock.sh.j2 | 39 ++++++++++++++------------ templates/tuned.conf.j2 | 2 +- 3 files changed, 57 insertions(+), 37 deletions(-) diff --git a/playbook.yml b/playbook.yml index 9388c42..c7e96a6 100644 --- a/playbook.yml +++ b/playbook.yml @@ -6,35 +6,48 @@ # 0.5 = 50% # 1.0 = 100% (of card power capability, not stock limits) # should not exceed 1.0, must be a float. driver will do some rounding/stepping - power_max_custom_multi: 1.0 # used to control the effective power limit in the 'custom' AMDGPU tuned profile - power_max_default_multi: 0.75 # used to limit GPU power to some lower percentage on default perf mode - # minimum/maximum GPU clocks using 'powerplay'. + # default is ~87% (281/323) on my 6900XT. check '/sys/class/drm/card*/device/hwmon/hwmon*/power1_cap_default' + power_max_custom_multi: 0.928793 # calculated to ~300W. used to control the effective power limit in the non-default AMDGPU tuned profiles + power_max_default_multi: 0.82 # (typically) used to limit GPU power to some lower percentage on default perf mode/profile + # + # minimum/maximum GPU clocks using 'powerplay' + # these do not apply to the resulting 'amdgpu-default' tuned profile + # only the others (eg: 'VR'/'custom') under amdgpu_profiles (below) # effective clocks are dynamically adjusted by the driver in this range - based on utilization # can find baseline values in the '/sys/class/drm/{{ card }}/device/pp_od_clk_voltage' file # OD_RANGE indicates acceptable value ranges for SCLK (core) and MCLK (memory) # Under 'OD_SCLK' and 'OD_MCLK' you will see 0/1. These are the minimum/maximum values for the respective component. - gpu_clock_min: "2300" - gpu_clock_max: "2600" + gpu_clock_min: "2000" + gpu_clock_max: "2615" + # # memory clocks are static, we only set a 'max' - high-refresh rate display quirk # this allows the memory clock to be adjusted gpumem_clock_max: "1075" + # # optional, applies offset to GPU voltage, eg: '+100' = to boost GPU core voltage 100mV or 0.1V. for the 'custom' GPU profile. - gpu_mv_offset: "+50" + gpu_mv_offset: "-25" + # # the card for 'tuned' to control # this is *usually* 'card0', but may differ. # check '/dev/dri/card*' card: 'card0' # default to card0 - base_profiles: # list of source tuned profiles available on Fedora (TODO: should dynamically discover?) - - balanced # these are further modified with secondary (templated) profiles, see 'templates/tuned.conf.j2' + # list of source tuned profiles available on Fedora (TODO: should dynamically discover) + # further modified with AMD GPU power/clock parameters, creating new profiles. eg: 'balanced-amdgpu-VR' + base_profiles: + - balanced - desktop - latency-performance - network-latency - network-throughput - powersave - virtual-host - amdgpu_profiles: # statically defined mapping of the contents in /sys/class/drm/{{ card }}/device/pp_power_profile_mode - - { name: 'default', value: 0 } - - { name: 'custom', value: 6 } + amdgpu_profiles: # statically defined mapping of the contents in /sys/class/drm/card*/device/pp_power_profile_mode + default: # more may be added, but do not remove default/custom. new profiles require a script template, see 'templates' + pwrmode: 0 + VR: + pwrmode: 4 + custom: + pwrmode: 6 handlers: - name: restart tuned ansible.builtin.service: @@ -85,30 +98,34 @@ - name: create custom profile directories file: state: directory - path: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.name }} + path: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }} mode: "0755" with_nested: - - "{{ amdgpu_profiles }}" + - "{{ lookup('dict', amdgpu_profiles) }}" - "{{ base_profiles }}" - - name: template AMDGPU clock control scripts (default/GPU profile dependency) + - name: template AMDGPU control/reset scripts template: src: templates/amdgpu-clock.sh.j2 - dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.name }}/amdgpu-clock.sh + dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }}/amdgpu-clock.sh owner: root group: root mode: "0755" with_nested: - - "{{ amdgpu_profiles }}" + - "{{ lookup('dict', amdgpu_profiles) }}" - "{{ base_profiles }}" notify: restart tuned - name: template custom tuned profiles template: src: templates/tuned.conf.j2 - dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.name }}/tuned.conf + dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }}/tuned.conf owner: root group: root mode: "0644" with_nested: - - "{{ amdgpu_profiles }}" + - "{{ lookup('dict', amdgpu_profiles) }}" - "{{ base_profiles }}" notify: restart tuned + - name: ensure tuned is enabled + service: + name: tuned + enabled: yes diff --git a/templates/amdgpu-clock.sh.j2 b/templates/amdgpu-clock.sh.j2 index 34d3634..2f5ec2b 100644 --- a/templates/amdgpu-clock.sh.j2 +++ b/templates/amdgpu-clock.sh.j2 @@ -3,42 +3,46 @@ # configures GPU power/clock characteristics # clocks/power in 3D are dynamic based on need/usage # +# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default +# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR) +# # rendered by Ansible with environment-appropriate values: # card #, eg: card0 # path to discovered sysfs device files (power/clock/voltage control) # -# this sets the minimums / maximums for a specific generation of GPU -# settings may need adjusted -# # AMDGPU driver/sysfs references: # https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html # https://docs.kernel.org/gpu/amdgpu/thermal.html +{# begin the templated script for 'default' profiles to reset state #} +{% if 'default' in item.0.key %} -{% if 'default' in item.0.name %} -# set power state transition heuristics to default -echo '{{ item.0.value }}' | tee /sys/class/drm/{{ card }}/device/pp_power_profile_mode +# set power state transition heuristics to default +echo '{{ item.0.value.pwrmode }}' | tee /sys/class/drm/{{ card }}/device/pp_power_profile_mode # set control mode back to auto -# attempts to dynamically set optimal power profile for conditions +# attempts to dynamically set optimal power profile for (load) conditions echo 'auto' | tee /sys/class/drm/{{ card }}/device/power_dpm_force_performance_level # reset any existing profile clock changes echo 'r' | tee /sys/class/drm/{{ card }}/device/pp_od_clk_voltage # give default profile {{ power_max_default_multi * 100.0 |int }}% of the max power capability -# {{ power_cap_default|int/1000000 }} Watts of {{ power_max|int/1000000 }} total +# {{ power_cap_default|int/1000000 }} Watts of {{ power_max|int/1000000 }} total echo '{{ power_cap_default }}' | tee {{ powercap_set.files.0.path }} -{% elif 'custom' in item.0.name %} +{% else %} + +{# begin the templated script for non-default AMD GPU profiles, eg: 'VR' or '3D_FULL_SCREEN' #} # set manual control mode -# allow control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', and 'pp_power_profile_mode' files +# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files +# only interested in 'pp_power_profile_mode' for power and 'pp_dpm_mclk' for memory clock (flickering). +# GPU clocks are dynamic based on (load) condition echo 'manual' | tee /sys/class/drm/{{ card }}/device/power_dpm_force_performance_level -# set power state transition heuristics to custom/manual -# looked up from amdgpu_profiles variable using 'with_nested' loop in task -echo '{{ item.0.value }}' | tee /sys/class/drm/{{ card }}/device/pp_power_profile_mode +# set power state transition heuristics to '{{ item.0.key }}' profile +echo '{{ item.0.value.pwrmode }}' | tee /sys/class/drm/{{ card }}/device/pp_power_profile_mode # give this profile {{ power_max_custom_multi * 100.0 |int }}% of the max power capability -# {{ power_cap_custom|int/1000000 }} Watts of {{ power_max|int/1000000 }} total +# {{ power_cap_custom|int/1000000 }} Watts of {{ power_max|int/1000000 }} total echo '{{ power_cap_custom }}' | tee {{ powercap_set.files.0.path }} # set the minimum GPU clock @@ -58,9 +62,8 @@ echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/{{ card }}/device/pp_od_clk_v # commit the changes echo 'c' | tee /sys/class/drm/{{ card }}/device/pp_od_clk_voltage -# force GPU core/shader into the highest profile: {{ gpu_clock_max }}Mhz -echo '1' | tee /sys/class/drm/{{ card }}/device/pp_dpm_sclk - -# force GPU memory into highest profile +# force GPU memory into highest clock (fix flickering) +# pp_dpm_*clk settings are unintuitive, giving profiles that may be used +# opt not to set the others (eg: sclk/fclk) - those should remain for benefits from the curve echo '3' | tee /sys/class/drm/{{ card }}/device/pp_dpm_mclk {% endif %} diff --git a/templates/tuned.conf.j2 b/templates/tuned.conf.j2 index 8e3e9ca..636abad 100644 --- a/templates/tuned.conf.j2 +++ b/templates/tuned.conf.j2 @@ -1,6 +1,6 @@ [main] include={{ item.1 }} -summary={{ item.1 }} + TCP/RAID tweaks + AMDGPU pp_power_profile_mode = {{ item.0.value }} ({{ item.0.name }}) +summary={{ item.1 }} + TCP/RAID tweaks + AMDGPU pp_power_profile_mode = {{ item.0.value.pwrmode }} ({{ item.0.key }}) [sysctl] net.core.default_qdisc=fq