From 7c50e771e20b670484e1b5c734ef263f17b1f119 Mon Sep 17 00:00:00 2001
From: Josh Lay <me@jlay.io>
Date: Fri, 7 Jul 2023 23:45:36 -0500
Subject: [PATCH] refactor/clean up, move to separate templates per profile

---
 playbook.yml                                  | 11 ++--
 roles/tuned_amdgpu/defaults/main.yml          |  1 +
 roles/tuned_amdgpu/files/profile-common.sh    | 35 +++++++++++
 roles/tuned_amdgpu/tasks/main.yml             | 31 ++++++++--
 .../templates/amdgpu-profile-default.sh.j2    | 36 ++++++++++++
 .../templates/amdgpu-profile-overclock.sh.j2  | 58 +++++++++++++++++++
 ...-clock.sh.j2 => amdgpu-profile-peak.sh.j2} | 50 +---------------
 7 files changed, 164 insertions(+), 58 deletions(-)
 create mode 100644 roles/tuned_amdgpu/files/profile-common.sh
 create mode 100644 roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2
 create mode 100644 roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2
 rename roles/tuned_amdgpu/templates/{amdgpu-clock.sh.j2 => amdgpu-profile-peak.sh.j2} (61%)

diff --git a/playbook.yml b/playbook.yml
index ec0ffab..eec99dd 100644
--- a/playbook.yml
+++ b/playbook.yml
@@ -11,16 +11,17 @@
       # the connected AMD GPU is automatically discovered - assumes one
       # on swap to other AMD cards to avoid instability:
       #   'rm -rfv /etc/tuned/*amdgpu*'
-      gpu_clock_min: "2200" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile
-      gpu_clock_max: "2725" # default somewhere around 2529 to 2660
+      gpu_clock_min: "750" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile
+      gpu_clock_max: "2675" # default somewhere around 2529 to 2660
       gpumem_clock_static: "1075"
       gpu_power_multi:
         default: 0.869969040247678 # 281W - real default
-#        overclock: 0.928792569659443 # 300W - slight boost
-        overclock: 1.0 # 323W - full board capability
+        overclock: 0.928792569659443 # 300W - slight boost
+#        overclock: 1.0 # 323W - full board capability
       # optional, applies offset (+/-) to GPU voltage by provided mV
       # gpu_mv_offset: "-25"
-      gu_mv_offset: "+75" # add 50mV or 0.075V
+      # gpu_mv_offset: "+50" # add 50mV or 0.05V
+      gpu_mv_offset: "+25" # add 25mV or 0.025V
       # '-50' undervolts GPU core voltage 50mV or 0.05V
       # mostly untested, there be dragons/instability
       #
diff --git a/roles/tuned_amdgpu/defaults/main.yml b/roles/tuned_amdgpu/defaults/main.yml
index 14a9581..6792dcb 100644
--- a/roles/tuned_amdgpu/defaults/main.yml
+++ b/roles/tuned_amdgpu/defaults/main.yml
@@ -9,3 +9,4 @@ profile_name: "{{ item.0 }}"
 amdgpu_profiles:
   - default
   - overclock
+  - peak
diff --git a/roles/tuned_amdgpu/files/profile-common.sh b/roles/tuned_amdgpu/files/profile-common.sh
new file mode 100644
index 0000000..5970513
--- /dev/null
+++ b/roles/tuned_amdgpu/files/profile-common.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# 'common' file sourced by other scripts under tuned profile
+#
+# dynamically determine the connected GPU using the DRM subsystem
+CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1)
+
+function get_hwmon_dir() {
+    CARD_DIR="/sys/class/drm/${1}/device/"
+    for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do
+            if [[ -f "${CANDIDATE}"/power1_cap ]]; then
+                    # found a valid hwmon dir
+                    echo "${CANDIDATE}"
+            fi
+    done
+}
+
+# determine the hwmon directory
+HWMON_DIR=$(get_hwmon_dir "${CARD}")
+
+# read all of the power profiles, used to get the IDs for assignment later
+PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode)
+
+# get power capability; later used determine limits
+read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max
+
+# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl
+# improves allocation latency
+echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled
+
+# export determinations
+export CARD
+export HWMON_DIR
+export PROFILE_MODES
+export POWER_CAP
diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml
index a7bc037..4cc4e42 100644
--- a/roles/tuned_amdgpu/tasks/main.yml
+++ b/roles/tuned_amdgpu/tasks/main.yml
@@ -28,6 +28,14 @@
   when: (fed_ppdtuned_swap is not defined) or ('tuned' not in ansible_facts.packages)
   become: true
 
+- name: Ensure dynamic tuning is disabled
+  ansible.builtin.lineinfile:
+    path: /etc/tuned/tuned-main.conf
+    regexp: '^dynamic_tuning.*='
+    line: 'dynamic_tuning = 0'
+  notify: Restart tuned
+  become: true
+
 - name: Create custom profile directories
   ansible.builtin.file:
     state: directory
@@ -38,20 +46,31 @@
     - "{{ base_profiles }}"
   become: true
 
-- name: Template AMDGPU control/reset scripts
+- name: Copy 'common' AMDGPU script for all profiles
+  ansible.builtin.copy:
+    src: profile-common.sh
+    dest: "/etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-common.sh"
+    mode: "0644"  # sourced, doesn't require executable bit
+    owner: root
+    group: root
+  notify: Restart tuned
+  with_nested:
+    - "{{ amdgpu_profiles }}"
+    - "{{ base_profiles }}"
+  become: true
+
+- name: Template custom AMDGPU profile scripts
   ansible.builtin.template:
-    src: templates/amdgpu-clock.sh.j2
+    src: amdgpu-profile-{{ item.0 }}.sh.j2
     dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-clock.sh
     owner: root
     group: root
     mode: "0755"
-  with_nested:
-    - "{{ amdgpu_profiles }}"
-    - "{{ base_profiles }}"
+  loop: "{{ amdgpu_profiles | product(base_profiles) | list }}"
   notify: Restart tuned
   become: true
 
-- name: Template custom tuned profiles
+- name: Template tuned.conf for custom profiles
   ansible.builtin.template:
     src: templates/tuned.conf.j2
     dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/tuned.conf
diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2
new file mode 100644
index 0000000..4bd1282
--- /dev/null
+++ b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2
@@ -0,0 +1,36 @@
+#!/bin/bash
+# script for tuned AMDGPU clock control
+# configures GPU power/clock characteristics
+# clocks/power in 3D are dynamic based on need/usage
+#
+# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default
+# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR)
+#
+# rendered by Ansible with environment-appropriate values:
+#   card #, eg: card0
+#   path to discovered sysfs device files (power/clock/voltage control)
+#
+# AMDGPU driver/sysfs references:
+#   https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html
+#   https://docs.kernel.org/gpu/amdgpu/thermal.html
+#
+# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability
+. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh
+
+{# begin the templated script for 'default' profiles to reset state #}
+# set control mode back to auto
+#   attempts to dynamically set optimal power profile for (load) conditions
+echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level
+
+# reset any existing profile clock changes
+echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+
+# adjust power limit using multiplier against board capability
+POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}')
+echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap"
+
+# extract the power-saving profile ID number
+PROF_DEFAULT_NUM=$(/usr/bin/awk '$0 ~ /BOOTUP_DEFAULT.*:/ {print $1}' <<< "$PROFILE_MODES")
+
+# reset power/clock heuristics to power-saving
+echo "${PROF_DEFAULT_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode
diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2
new file mode 100644
index 0000000..1f0aa3a
--- /dev/null
+++ b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2
@@ -0,0 +1,58 @@
+#!/bin/bash
+# script for tuned AMDGPU clock control
+# configures GPU power/clock characteristics
+# clocks/power in 3D are dynamic based on need/usage
+#
+# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default
+# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR)
+#
+# rendered by Ansible with environment-appropriate values:
+#   card #, eg: card0
+#   path to discovered sysfs device files (power/clock/voltage control)
+#
+# AMDGPU driver/sysfs references:
+#   https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html
+#   https://docs.kernel.org/gpu/amdgpu/thermal.html
+#
+# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability
+. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh
+
+{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #}
+# set the minimum GPU clock - for best performance, this should be near the maximum
+# RX6000 series power management *sucks*
+echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+
+# set the maximum GPU clock
+echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+
+# set the GPU *memory* clock
+# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range
+# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow
+echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+{% if gpu_mv_offset is defined %}
+
+# offset GPU voltage {{ gpu_mv_offset }}mV
+echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+{% endif %}
+
+# commit the changes
+echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
+
+# force GPU core and memory into highest clocks (fix flickering and poor power management)
+# set manual control mode
+#   allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files
+echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level
+
+# adjust power limit using multiplier against board capability
+POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}')
+echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap"
+
+# avoid display flickering, force OC'd memory to highest clock
+echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk
+
+# extract the VR power profile ID number
+PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES")
+
+# force 'overclocked' profile to 'VR' power/clock heuristics
+# latency/frame timing seemed favorable with relatively-close minimum clocks
+echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode
diff --git a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2
similarity index 61%
rename from roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2
rename to roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2
index cc2dd2a..14105a8 100644
--- a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2
+++ b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2
@@ -13,53 +13,10 @@
 # AMDGPU driver/sysfs references:
 #   https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html
 #   https://docs.kernel.org/gpu/amdgpu/thermal.html
+#
+# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability
+. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh
 
-{# done this way to avoid issues with the card number possibly shifting after playbook run #}
-# dynamically determine the connected GPU using the DRM subsystem
-CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1)
-
-function get_hwmon_dir() {
-    CARD_DIR="/sys/class/drm/${1}/device/"
-    for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do
-            if [[ -f "${CANDIDATE}"/power1_cap ]]; then
-                    # found a valid hwmon dir
-                    echo "${CANDIDATE}"
-            fi
-    done
-}
-
-# determine the hwmon directory
-HWMON_DIR=$(get_hwmon_dir "${CARD}")
-
-# read all of the power profiles, used to get the IDs for assignment later
-PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode)
-
-# get power capability; later used determine limits
-read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max
-
-# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl
-# improves allocation latency
-echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled
-
-{# begin the templated script for 'default' profiles to reset state #}
-{% if 'default' in profile_name %}
-# set control mode back to auto
-#   attempts to dynamically set optimal power profile for (load) conditions
-echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level
-
-# reset any existing profile clock changes
-echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage
-
-# adjust power limit using multiplier against board capability
-POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}')
-echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap"
-
-# extract the power-saving profile ID number
-PROF_POWER_SAVING_NUM=$(/usr/bin/awk '$0 ~ /POWER_SAVING.*:/ {print $1}' <<< "$PROFILE_MODES")
-
-# reset power/clock heuristics to power-saving
-echo "${PROF_POWER_SAVING_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode
-{% else %}
 {# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #}
 # set the minimum GPU clock - for best performance, this should be near the maximum
 # RX6000 series power management *sucks*
@@ -107,4 +64,3 @@ echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mod
 # ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500
 # followup: doesn't work that well in practice, still flaky on clocks/frame times
 #echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level
-{% endif %}