From e15ff1af45959ea3ee845bd3f448d04d4f8a321b Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sun, 27 Nov 2022 01:26:49 -0600 Subject: [PATCH 01/10] make tuned handler escalate privs --- roles/tuned_amdgpu/handlers/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/tuned_amdgpu/handlers/main.yml b/roles/tuned_amdgpu/handlers/main.yml index 60384eb..c9a9ad5 100644 --- a/roles/tuned_amdgpu/handlers/main.yml +++ b/roles/tuned_amdgpu/handlers/main.yml @@ -4,3 +4,4 @@ ansible.builtin.service: name: tuned state: restarted + become: true From 4f6314f27039781bc68d362dc576792bad8f9250 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sat, 28 Jan 2023 22:02:20 -0600 Subject: [PATCH 02/10] adjust preset clock/power multipliers slightly --- playbook.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/playbook.yml b/playbook.yml index 76d2f95..d47f7bd 100644 --- a/playbook.yml +++ b/playbook.yml @@ -7,9 +7,9 @@ - role: tuned_amdgpu # note: 'gpu_*' vars only apply with the 'custom' suffixed profiles created by this tooling # profiles based on the 'default' amdgpu power profile mode use default clocks - gpu_clock_min: "750" # default 500 + gpu_clock_min: "500" # default 500 gpu_clock_max: "2600" # default 2529 - gpumem_clock_static: "1050" + gpumem_clock_static: "1075" # optional, applies offset (+/-) to GPU voltage by provided mV gpu_mv_offset: "-50" # '-50' undervolts GPU core voltage 50mV or 0.05V @@ -32,18 +32,21 @@ amdgpu_profiles: default: pwrmode: 0 - pwr_cap_multi: 0.789473684210526 # 255W - default + pwr_cap_multi: 0.789473684210526 # 255W - slightly reduced, maintains clocks well 3D: pwrmode: 1 - pwr_cap_multi: 0.789473684210526 # 255W - default + pwr_cap_multi: 0.869969040247678 # 281W - default + powersave: + pwrmode: 2 + pwr_cap_multi: 0.869969040247678 VR: pwrmode: 4 - pwr_cap_multi: 0.789473684210526 # 255W - default + pwr_cap_multi: 0.869969040247678 compute: pwrmode: 5 - pwr_cap_multi: 0.789473684210526 # 255W - default + pwr_cap_multi: 0.869969040247678 custom: pwrmode: 6 - pwr_cap_multi: 0.869969040247678 # 281W - slight boost + pwr_cap_multi: 1.0 # 323W - full capability # both dictionaries are merged to create new 'tuned' profiles. eg: # 'balanced-amdgpu-default', 'balanced-amdgpu-3D', 'balanced-amdgpu-video' From f2e4923658f7c6533fd206ea20229513be603b01 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sat, 28 Jan 2023 22:03:03 -0600 Subject: [PATCH 03/10] tuned template: set "none" I/O scheduler for non-rotational devices --- roles/tuned_amdgpu/templates/tuned.conf.j2 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/roles/tuned_amdgpu/templates/tuned.conf.j2 b/roles/tuned_amdgpu/templates/tuned.conf.j2 index 636abad..7ed559b 100644 --- a/roles/tuned_amdgpu/templates/tuned.conf.j2 +++ b/roles/tuned_amdgpu/templates/tuned.conf.j2 @@ -20,3 +20,9 @@ vm.max_map_count=1048576 [gpuclockscript] type=script script=${i:PROFILE_DIR}/amdgpu-clock.sh + +# for SSDs with no RPM, set no IO scheduler +[ssdnosched] +type=disk +devices_udev_regex=(ID_ATA_ROTATION_RATE_RPM=0) +elevator=none From bb03fc2cc2499152c68cad436956d48bbbedf6d5 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sat, 8 Apr 2023 12:28:51 -0500 Subject: [PATCH 04/10] modernize/simplify: OC assumes "high" perf profile --- README.md | 12 ++--- playbook.yml | 45 ++++++++++--------- roles/tuned_amdgpu/defaults/main.yml | 22 ++++++--- roles/tuned_amdgpu/tasks/main.yml | 12 ++--- .../tuned_amdgpu/templates/amdgpu-clock.sh.j2 | 29 ++++++------ roles/tuned_amdgpu/templates/tuned.conf.j2 | 8 +++- 6 files changed, 71 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 22deb92..d59c9e9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tuned-amdgpu -Hacky solution to integrate AMDGPU power profile control in `tuned` with Ansible +Hacky solution to integrate AMDGPU power control and overclocking in `tuned` with Ansible Takes a list of existing `tuned` profiles and creates new ones based on them. These new profiles include AMDGPU power/clock parameters @@ -10,7 +10,7 @@ $ grep -ls ^connected /sys/class/drm/*/status | grep -o card[0-9] | sort | uniq card1 ``` -_Warning_: This is only tested with `RX6000` series GPUs, it is probable that older AMD GPUs will not work properly. Use at your own risk! +_Warning_: This is only tested with `RX6000` series GPUs, it is probable that other generations will *not* work properly. Use at your own risk! ## Profiles @@ -19,10 +19,7 @@ An example of the output/provided profiles follow | Output profile | Description | |:---|---| | `balanced-amdgpu-default` | Includes the (assumed) existing `balanced` tuned profile.

Only adjusts the GPU power limit (typically lower). Clocks/voltage curve remain the default. | -| `desktop-amdgpu-VR` | Includes the (assumed) existing `desktop` tuned profile.

Adjusts the GPU power limit, clocks, _and_ the voltage curve.

Uses the predefined `VR` profile in the driver. See `/sys/class/drm/card*/device/pp_power_profile_mode` | -| `latency-performance-amdgpu-custom` | Includes the existing `latency-performance` tuned profile.

Like the existing GPU profiles (eg: _VR)), this also adjusts the GPU power limit, clocks, _and_ the voltage curve.

This differs by using the `custom` profile in the driver. This opens up further tweaking of the power/clock heuristics through the driver (currently manual). see: [pp-dpm](https://docs.kernel.org/gpu/amdgpu/thermal.html#pp-dpm) | - -**Note**: This is non-exhaustive, see the variables `base_profiles` and `amdgpu_profiles` below for the (default) sources of the merged profile mapping +| `desktop-amdgpu-overclock` | Includes the (assumed) existing `desktop` tuned profile.

Adjusts the GPU power limit, clocks, _and_ the voltage curve. | ## Notable variables @@ -35,5 +32,4 @@ These are the variables you're likely to want to change. They are defined in [p | gpumem_clock_static | Sets the _static_ memory clock for the GPU (in `MHz`). This is *not* the _effective_ data rate. That is a multiple of this depending on the type of VRAM.

To avoid flickering this does *not* change dynamically with load. | `1050`, results in just over `1GHz`; mild overclock

Actual effective clock depends on this being multiplied against the data/pump rate of the `GDDR?` GPU memory | | gpu_mv_offset | GPU core voltage offset. Takes +/- some integer in millivolts. Can be used to both over _and_ under volt. | `-50` (undervolt `50mV` or `0.05V`) | | base_profiles | List of base tuned profiles to clone in the new AMDGPU profiles. Defaults based on `Fedora` |
  • `balanced`
  • `desktop`
  • `latency-performance`
  • `network-latency`
  • `network-throughput`
  • `powersave`
  • `virtual-host`
    • | -| amdgpu_profiles | Dictionary mapping the AMDGPU power profiles found in `/sys/class/drm/card*/device/pp_power_profile_mode` and custom power limits.

    For each item, two keys: `pwrmode` and `pwr_cap_multi`.

    `pwrmode` maps to the number assigned in `/sys` above.
    `pwr_cap_multi` is a multiplier against board power capability. Must be a float, eg: `0.5` for *50%* | 
    default:
      pwrmode: 0
      pwr_cap_multi: 0.75
      # 75% relatively safe default
    VR:
      pwrmode: 4
      pwr_cap_multi: 0.8
      # 80%, likely slight boost
    custom:
      pwrmode: 6
      pwr_cap_multi: 1.0
      # 100%, full GPU board capability
      # warning: significantly increased heat
    | - +| gpu_power_multi | Dictionary with two keys, `default` and `overclock`. Expects two floats to set a power limit relative to the board _capability_. Example: `1.0` is full board capability, `0.5` is 50%.| diff --git a/playbook.yml b/playbook.yml index d47f7bd..e710f7b 100644 --- a/playbook.yml +++ b/playbook.yml @@ -8,10 +8,13 @@ # note: 'gpu_*' vars only apply with the 'custom' suffixed profiles created by this tooling # profiles based on the 'default' amdgpu power profile mode use default clocks gpu_clock_min: "500" # default 500 - gpu_clock_max: "2600" # default 2529 + gpu_clock_max: "2615" # default 2529 gpumem_clock_static: "1075" + gpu_power_multi: + default: 0.789473684210526 # 255W - slightly reduced, maintains clocks well + overclock: 0.869969040247678 # 281W - real default, board supports up to 323W (1.0) # optional, applies offset (+/-) to GPU voltage by provided mV - gpu_mv_offset: "-50" + # gpu_mv_offset: "-25" # '-50' undervolts GPU core voltage 50mV or 0.05V # # list of source tuned profiles available on Fedora (TODO: should dynamically discover) @@ -29,24 +32,24 @@ # ref: https://www.kernel.org/doc/html/v4.20/gpu/amdgpu.html#pp-power-profile-mode # 'pwr_cap_multi' is multiplied against board *limit* to determine profile wattage; 0.5 = 50% # values below reflect my 6900XT - amdgpu_profiles: - default: - pwrmode: 0 - pwr_cap_multi: 0.789473684210526 # 255W - slightly reduced, maintains clocks well - 3D: - pwrmode: 1 - pwr_cap_multi: 0.869969040247678 # 281W - default - powersave: - pwrmode: 2 - pwr_cap_multi: 0.869969040247678 - VR: - pwrmode: 4 - pwr_cap_multi: 0.869969040247678 - compute: - pwrmode: 5 - pwr_cap_multi: 0.869969040247678 - custom: - pwrmode: 6 - pwr_cap_multi: 1.0 # 323W - full capability +# amdgpu_profiles: +# default: +# pwrmode: 0 +# pwr_cap_multi: 0.789473684210526 # 255W - slightly reduced, maintains clocks well +# 3D: +# pwrmode: 1 +# pwr_cap_multi: 0.869969040247678 # 281W - default +# powersave: +# pwrmode: 2 +# pwr_cap_multi: 0.869969040247678 +# VR: +# pwrmode: 4 +# pwr_cap_multi: 0.869969040247678 +# compute: +# pwrmode: 5 +# pwr_cap_multi: 0.869969040247678 +# custom: +# pwrmode: 6 +# pwr_cap_multi: 1.0 # 323W - full capability # both dictionaries are merged to create new 'tuned' profiles. eg: # 'balanced-amdgpu-default', 'balanced-amdgpu-3D', 'balanced-amdgpu-video' diff --git a/roles/tuned_amdgpu/defaults/main.yml b/roles/tuned_amdgpu/defaults/main.yml index de80a28..aa86c15 100644 --- a/roles/tuned_amdgpu/defaults/main.yml +++ b/roles/tuned_amdgpu/defaults/main.yml @@ -8,8 +8,20 @@ board_watts: "{{ power_max | int / 1000000 }}" # internals for profile power calculations # item in the context of the with_nested loops in the play -profile_name: "{{ item.0.key }}" -profile_percentage: "{{ (item.0.value.pwr_cap_multi * 100.0) | round(2) }}" -profile_multi: "{{ item.0.value.pwr_cap_multi }}" -profile_microwatts: "{{ power_max | float * profile_multi | float }}" -profile_watts: "{{ profile_microwatts | int / 1000000 }}" +profile_name: "{{ item.0 }}" + +# determine percentage for human-friendly comments +power_default_pct: "{{ (gpu_power_multi.default * 100.0) | round(2) }}" +power_oc_pct: "{{ (gpu_power_multi.overclock * 100.0) | round(2) }}" + +# in microWatts, actually written to sysfs +power_default_mw: "{{ (power_max | float) * (gpu_power_multi.default | float) }}" +power_oc_mw: "{{ (power_max | float) * (gpu_power_multi.overclock | float) }}" + +# wattages - more human-friendly comments +power_default_watts: "{{ (power_default_mw | int) / 1000000 }}" +power_oc_watts: "{{ (power_oc_mw | int) / 1000000 }}" + +amdgpu_profiles: + - default + - overclock diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml index 5f93274..c428dda 100644 --- a/roles/tuned_amdgpu/tasks/main.yml +++ b/roles/tuned_amdgpu/tasks/main.yml @@ -63,22 +63,22 @@ - name: Create custom profile directories ansible.builtin.file: state: directory - path: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }} + path: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }} mode: "0755" with_nested: - - "{{ lookup('dict', amdgpu_profiles) }}" + - "{{ amdgpu_profiles }}" - "{{ base_profiles }}" become: true - name: Template AMDGPU control/reset scripts ansible.builtin.template: src: templates/amdgpu-clock.sh.j2 - dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }}/amdgpu-clock.sh + dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-clock.sh owner: root group: root mode: "0755" with_nested: - - "{{ lookup('dict', amdgpu_profiles) }}" + - "{{ amdgpu_profiles }}" - "{{ base_profiles }}" notify: Restart tuned become: true @@ -86,12 +86,12 @@ - name: Template custom tuned profiles ansible.builtin.template: src: templates/tuned.conf.j2 - dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0.key }}/tuned.conf + dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/tuned.conf owner: root group: root mode: "0644" with_nested: - - "{{ lookup('dict', amdgpu_profiles) }}" + - "{{ amdgpu_profiles }}" - "{{ base_profiles }}" notify: Restart tuned become: true diff --git a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 index 90c5f0b..28af80a 100644 --- a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 +++ b/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 @@ -20,9 +20,6 @@ CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o ' {# begin the templated script for 'default' profiles to reset state #} {% if 'default' in profile_name %} -# set power state transition heuristics to default -echo '{{ item.0.value.pwrmode }}' | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode - # set control mode back to auto # attempts to dynamically set optimal power profile for (load) conditions echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level @@ -30,23 +27,20 @@ echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_le # reset any existing profile clock changes echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -# give '{{ profile_name }}' profile ~{{ profile_percentage }}% (rounded) of the max power capability -# {{ profile_watts }} Watts of {{ board_watts }} total -echo '{{ profile_microwatts | int }}' | tee '{{ powercap_set.files.0.path }}' +# give '{{ profile_name }}' profile ~{{ power_default_pct }}% (rounded) of the max power capability +# {{ power_default_watts }} Watts of {{ board_watts }} total +echo '{{ power_default_mw | int }}' | tee '{{ powercap_set.files.0.path }}' {% else %} {# begin the templated script for non-default AMD GPU profiles, eg: 'VR' or '3D_FULL_SCREEN' #} # set manual control mode # allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files # only interested in 'pp_power_profile_mode' for power and 'pp_dpm_mclk' for memory clock (flickering). # GPU clocks are dynamic based on (load) condition -echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level +#echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level -# set power state transition heuristics to '{{ profile_name }}' profile -echo '{{ item.0.value.pwrmode }}' | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode - -# give '{{ profile_name }}' profile ~{{ profile_percentage }}% (rounded) of the max power capability -# {{ profile_watts }} Watts of {{ board_watts }} total -echo '{{ profile_microwatts | int }}' | tee '{{ powercap_set.files.0.path }}' +# give '{{ profile_name }}' profile ~{{ power_oc_pct }}% (rounded) of the max power capability +# {{ power_oc_watts }} Watts of {{ board_watts }} total +echo '{{ power_oc_mw | int }}' | tee '{{ powercap_set.files.0.path }}' # set the minimum GPU clock echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage @@ -54,7 +48,8 @@ echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_v # set the maximum GPU clock echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -# set the maximum GPU *memory* clock +# set the minimum / maximum GPU *memory* clock - force it high +echo 'm 0 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage {% if gpu_mv_offset is defined %} @@ -68,5 +63,9 @@ echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage # force GPU memory into highest clock (fix flickering) # pp_dpm_*clk settings are unintuitive, giving profiles that may be used # opt not to set the others (eg: sclk/fclk) - those should remain for benefits from the curve -echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk +# echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk + +# note 4/8/2023: instead of 'manual'... deal with broken power management, force clocks to high +# ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500 +echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level {% endif %} diff --git a/roles/tuned_amdgpu/templates/tuned.conf.j2 b/roles/tuned_amdgpu/templates/tuned.conf.j2 index 7ed559b..ae98cd9 100644 --- a/roles/tuned_amdgpu/templates/tuned.conf.j2 +++ b/roles/tuned_amdgpu/templates/tuned.conf.j2 @@ -1,8 +1,10 @@ [main] include={{ item.1 }} -summary={{ item.1 }} + TCP/RAID tweaks + AMDGPU pp_power_profile_mode = {{ item.0.value.pwrmode }} ({{ item.0.key }}) +summary={{ item.1 }} + TCP/RAID tweaks + AMDGPU {{ item.0 }} [sysctl] +# allow regular users to see the kernel ring buffer +kernel.dmesg_restrict=0 net.core.default_qdisc=fq # 'bbr2' requires a [modified] supporting kernel - stock Fedora kernels do *not* support it (currently) # eg: 'kernel-xanmode-edge' from COPR 'rmnscnce/kernel-xanmod' @@ -25,4 +27,6 @@ script=${i:PROFILE_DIR}/amdgpu-clock.sh [ssdnosched] type=disk devices_udev_regex=(ID_ATA_ROTATION_RATE_RPM=0) -elevator=none +# elevator=none +elevator=kyber +# elevator=mq-deadline From c833a9a36e73918a7c709b36c498b8c98a0c4a5f Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sat, 3 Jun 2023 17:40:20 -0500 Subject: [PATCH 05/10] refactor, two profiles --- README.md | 5 +- playbook.yml | 42 +++------ roles/tuned_amdgpu/defaults/main.yml | 16 ---- roles/tuned_amdgpu/tasks/main.yml | 32 ------- .../tuned_amdgpu/templates/amdgpu-clock.sh.j2 | 85 ++++++++++++++----- roles/tuned_amdgpu/templates/tuned.conf.j2 | 8 +- 6 files changed, 82 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index d59c9e9..95fd9a8 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ Hacky solution to integrate AMDGPU power control and overclocking in `tuned` wit Takes a list of existing `tuned` profiles and creates new ones based on them. These new profiles include AMDGPU power/clock parameters An attempt is made to discover the active GPU using the 'connected' state in the `DRM` subsystem, example: -``` -$ grep -ls ^connected /sys/class/drm/*/status | grep -o card[0-9] | sort | uniq | sort -h | tail -1 + +```bash +~ $ grep -ls ^connected /sys/class/drm/*/status | grep -o card[0-9] | sort | uniq | sort -h | tail -1 card1 ``` diff --git a/playbook.yml b/playbook.yml index e710f7b..ec0ffab 100644 --- a/playbook.yml +++ b/playbook.yml @@ -7,15 +7,22 @@ - role: tuned_amdgpu # note: 'gpu_*' vars only apply with the 'custom' suffixed profiles created by this tooling # profiles based on the 'default' amdgpu power profile mode use default clocks - gpu_clock_min: "500" # default 500 - gpu_clock_max: "2615" # default 2529 + # + # the connected AMD GPU is automatically discovered - assumes one + # on swap to other AMD cards to avoid instability: + # 'rm -rfv /etc/tuned/*amdgpu*' + gpu_clock_min: "2200" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile + gpu_clock_max: "2725" # default somewhere around 2529 to 2660 gpumem_clock_static: "1075" gpu_power_multi: - default: 0.789473684210526 # 255W - slightly reduced, maintains clocks well - overclock: 0.869969040247678 # 281W - real default, board supports up to 323W (1.0) + default: 0.869969040247678 # 281W - real default +# overclock: 0.928792569659443 # 300W - slight boost + overclock: 1.0 # 323W - full board capability # optional, applies offset (+/-) to GPU voltage by provided mV # gpu_mv_offset: "-25" + gu_mv_offset: "+75" # add 50mV or 0.075V # '-50' undervolts GPU core voltage 50mV or 0.05V + # mostly untested, there be dragons/instability # # list of source tuned profiles available on Fedora (TODO: should dynamically discover) base_profiles: @@ -26,30 +33,3 @@ - network-throughput - powersave - virtual-host - # - # mapping of typical Navi generation power profiles from: - # /sys/class/drm/card*/device/pp_power_profile_mode - # ref: https://www.kernel.org/doc/html/v4.20/gpu/amdgpu.html#pp-power-profile-mode - # 'pwr_cap_multi' is multiplied against board *limit* to determine profile wattage; 0.5 = 50% - # values below reflect my 6900XT -# amdgpu_profiles: -# default: -# pwrmode: 0 -# pwr_cap_multi: 0.789473684210526 # 255W - slightly reduced, maintains clocks well -# 3D: -# pwrmode: 1 -# pwr_cap_multi: 0.869969040247678 # 281W - default -# powersave: -# pwrmode: 2 -# pwr_cap_multi: 0.869969040247678 -# VR: -# pwrmode: 4 -# pwr_cap_multi: 0.869969040247678 -# compute: -# pwrmode: 5 -# pwr_cap_multi: 0.869969040247678 -# custom: -# pwrmode: 6 -# pwr_cap_multi: 1.0 # 323W - full capability - # both dictionaries are merged to create new 'tuned' profiles. eg: - # 'balanced-amdgpu-default', 'balanced-amdgpu-3D', 'balanced-amdgpu-video' diff --git a/roles/tuned_amdgpu/defaults/main.yml b/roles/tuned_amdgpu/defaults/main.yml index aa86c15..14a9581 100644 --- a/roles/tuned_amdgpu/defaults/main.yml +++ b/roles/tuned_amdgpu/defaults/main.yml @@ -1,27 +1,11 @@ --- # defaults file for tuned_amdgpu # -# vars handling unit conversion RE: power capabilities/limits -# the discovered board limit for power capability; in microWatts, then converted -power_max: "{{ power_max_b64['content'] | b64decode }}" -board_watts: "{{ power_max | int / 1000000 }}" # internals for profile power calculations # item in the context of the with_nested loops in the play profile_name: "{{ item.0 }}" -# determine percentage for human-friendly comments -power_default_pct: "{{ (gpu_power_multi.default * 100.0) | round(2) }}" -power_oc_pct: "{{ (gpu_power_multi.overclock * 100.0) | round(2) }}" - -# in microWatts, actually written to sysfs -power_default_mw: "{{ (power_max | float) * (gpu_power_multi.default | float) }}" -power_oc_mw: "{{ (power_max | float) * (gpu_power_multi.overclock | float) }}" - -# wattages - more human-friendly comments -power_default_watts: "{{ (power_default_mw | int) / 1000000 }}" -power_oc_watts: "{{ (power_oc_mw | int) / 1000000 }}" - amdgpu_profiles: - default - overclock diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml index c428dda..a7bc037 100644 --- a/roles/tuned_amdgpu/tasks/main.yml +++ b/roles/tuned_amdgpu/tasks/main.yml @@ -28,38 +28,6 @@ when: (fed_ppdtuned_swap is not defined) or ('tuned' not in ansible_facts.packages) become: true -- name: Determine GPU device in drm subsystem - ansible.builtin.shell: - cmd: grep -ls ^connected /sys/class/drm/*/status | grep -o card[0-9] | sort | uniq | sort -h | tail -1 - executable: /bin/bash - changed_when: false - register: card - -- name: Find hwmon/max power capability file for {{ card.stdout }} - ansible.builtin.find: - paths: /sys/class/drm/{{ card.stdout }}/device/hwmon - file_type: file - recurse: true - use_regex: true - patterns: - - '^power1_cap_max$' - register: hwmon - -- name: Find hwmon/current power limit file for {{ card.stdout }} - ansible.builtin.find: - paths: /sys/class/drm/{{ card.stdout }}/device/hwmon - file_type: file - recurse: true - use_regex: true - patterns: - - '^power1_cap$' - register: powercap_set - -- name: Get max power capability for {{ card.stdout }} - ansible.builtin.slurp: - src: "{{ hwmon.files.0.path }}" - register: power_max_b64 - - name: Create custom profile directories ansible.builtin.file: state: directory diff --git a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 index 28af80a..cc2dd2a 100644 --- a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 +++ b/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 @@ -18,6 +18,29 @@ # dynamically determine the connected GPU using the DRM subsystem CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) +function get_hwmon_dir() { + CARD_DIR="/sys/class/drm/${1}/device/" + for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do + if [[ -f "${CANDIDATE}"/power1_cap ]]; then + # found a valid hwmon dir + echo "${CANDIDATE}" + fi + done +} + +# determine the hwmon directory +HWMON_DIR=$(get_hwmon_dir "${CARD}") + +# read all of the power profiles, used to get the IDs for assignment later +PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) + +# get power capability; later used determine limits +read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max + +# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl +# improves allocation latency +echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled + {# begin the templated script for 'default' profiles to reset state #} {% if 'default' in profile_name %} # set control mode back to auto @@ -27,29 +50,27 @@ echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_le # reset any existing profile clock changes echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -# give '{{ profile_name }}' profile ~{{ power_default_pct }}% (rounded) of the max power capability -# {{ power_default_watts }} Watts of {{ board_watts }} total -echo '{{ power_default_mw | int }}' | tee '{{ powercap_set.files.0.path }}' +# adjust power limit using multiplier against board capability +POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" + +# extract the power-saving profile ID number +PROF_POWER_SAVING_NUM=$(/usr/bin/awk '$0 ~ /POWER_SAVING.*:/ {print $1}' <<< "$PROFILE_MODES") + +# reset power/clock heuristics to power-saving +echo "${PROF_POWER_SAVING_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode {% else %} -{# begin the templated script for non-default AMD GPU profiles, eg: 'VR' or '3D_FULL_SCREEN' #} -# set manual control mode -# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files -# only interested in 'pp_power_profile_mode' for power and 'pp_dpm_mclk' for memory clock (flickering). -# GPU clocks are dynamic based on (load) condition -#echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# give '{{ profile_name }}' profile ~{{ power_oc_pct }}% (rounded) of the max power capability -# {{ power_oc_watts }} Watts of {{ board_watts }} total -echo '{{ power_oc_mw | int }}' | tee '{{ powercap_set.files.0.path }}' - -# set the minimum GPU clock +{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} +# set the minimum GPU clock - for best performance, this should be near the maximum +# RX6000 series power management *sucks* echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage # set the maximum GPU clock echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -# set the minimum / maximum GPU *memory* clock - force it high -echo 'm 0 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage +# set the GPU *memory* clock +# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range +# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage {% if gpu_mv_offset is defined %} @@ -60,12 +81,30 @@ echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_vo # commit the changes echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage -# force GPU memory into highest clock (fix flickering) -# pp_dpm_*clk settings are unintuitive, giving profiles that may be used -# opt not to set the others (eg: sclk/fclk) - those should remain for benefits from the curve -# echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk +# force GPU core and memory into highest clocks (fix flickering and poor power management) +# set manual control mode +# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files +echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level -# note 4/8/2023: instead of 'manual'... deal with broken power management, force clocks to high +# adjust power limit using multiplier against board capability +POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" + +# pp_dpm_*clk settings are unintuitive, giving profiles that may be used +echo '1' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_sclk +echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk +echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_fclk +echo '2' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_socclk + +# extract the VR power profile ID number +PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") + +# force 'overclocked' profile to 'VR' power/clock heuristics +# latency/frame timing seemed favorable with relatively-close minimum clocks +echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode + +# note 4/8/2023: instead of 'manual'... try dealing with broken power management, force clocks to high # ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500 -echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level +# followup: doesn't work that well in practice, still flaky on clocks/frame times +#echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level {% endif %} diff --git a/roles/tuned_amdgpu/templates/tuned.conf.j2 b/roles/tuned_amdgpu/templates/tuned.conf.j2 index ae98cd9..729e025 100644 --- a/roles/tuned_amdgpu/templates/tuned.conf.j2 +++ b/roles/tuned_amdgpu/templates/tuned.conf.j2 @@ -11,8 +11,12 @@ net.core.default_qdisc=fq net.ipv4.tcp_congestion_control=bbr2 net.core.rmem_max=33554432 net.core.wmem_max=33554432 -dev.raid.speed_limit_min=600000 -dev.raid.speed_limit_max=9000000 +dev.raid.speed_limit_min=1000000 +dev.raid.speed_limit_max=6000000 +# improve THP allocation latency, compact in background +vm.compaction_proactiveness=30 +# make page lock theft slightly more fair +vm.page_lock_unfairness=1 # allow some games to run (eg: DayZ) vm.max_map_count=1048576 From 5fdc4fe6a2b6124e08533cc90590f9037343a584 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Mon, 5 Jun 2023 01:00:36 -0500 Subject: [PATCH 06/10] update spreadsheet w/ my card multis --- power_max multi tab calculator.ods | Bin 18269 -> 22496 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/power_max multi tab calculator.ods b/power_max multi tab calculator.ods index de93923e200c8b12d9ec1b83f1ec32d16c9e7f1b..ea8c9b09f0f6161db4af3b59094b7f1c51b81df5 100644 GIT binary patch literal 22496 zcmb5Ubx@qavo5^2I|N-ML4$?hmc=~~2reN62<{Mk7YPz%A-H=22?_4*1PJcFNP@d9 zuX| zWcqIuboX?#^R)Hy`hS<`2Y=-z6Gpj3aoSMY+}iPd3yo z`M#e&vsIxYybiC+KC)u^6;FEi&`b@p4O{ec~{z&Jp^Sy}f% z$ysi!*zPWIr|z7^K(+FfCLVs>jr|l31_%_61p@tVANcRK??2vX>un|A=i(fntdo%a zjJ*Hkfjatlc42{&?UB_N=I_i6j$ht?S$q6$T(1!hrmrno??Ao$&?2X8<=PdZP&|OEzH0+gjgK8B9m`z_-us*)5UGF_3 zK)AvF?6^f>&v-K^o`;Qi-q@f}VVt2Q;p_)p-*VHHVbk}?_^Me728t4QuBzCGrkZz5 zPTy18jB`!hRGx4M(a|Z9XeUz(9xmG7@66fky|3-wTE_7iv*@yl;W?oU-}=@uVSTlR ze{noOesK{SQQKtRAZ^1RD^z{?Dk4wr;kuyYqn?td(tK+Q(U|fu<(TlWA)g&iMerTK}VIi9jHbwVSK=zbNYG;=F2p0AFsRL9dsRuP9HFuxdW`@e!_- zt8JBQ9cr-GtLu7f?9RUQ$dD38k2&z#XuZ&Ffs5zcvRaL3p~RH)ChKY3kQXT=iG_^J zzr_Dxl;>q#pq7bNO?GO%m=VL60%iZ=@%yWBvh@S>e)jTuJLj?s)8thi1u}KR z(|m^cu?5c3j5D_~?sz8q2Da#a=j1}_+LsirYVcYWU$2TpziUV&wYL{r0JA~r4 z&YE8Ih!#q3^+JesOKFk%bn%yP6D(}219ae6_e$d z8o$0eTh#sP^34Wwsak@tVF6!Xg6U*wz3B#jArIqcJQaggd{uUE{cjHMpmDsq)c0)h z7YZMwp6(qhMSr!ek8m~;SMG<}#077GEBPR6?JnEA0YeRj$<@=$Y$Yz*-1?NGy9+8; z57l-x{JF+OZUlCLSzbkJ9K9+DIt>9O9-$7>v+6ky&$VTAm$gNd2f12pG9wfBSPAl# zD2v#g;~_DB@v>4BbnOw?v2fvDQ?sLU%7CEe*FTr^&Z)No3S6kpzQMArLv;y#>|QnSvyH5}(Aq0m zu~(87R|t&8a=A<4_DdEj`=;jae>L6E^wNlS7h+%Xj2bj~dH(s!__6PlS70ZjiRlKj z?w*N2L?AVVFxx1Fh`n?_PLjc+8jIDB+&|I+eoH@U;jC2*i25c%kA(U**fa=fKW>)f z3>c22;;ZBJs(;UA9#X!{DOUD@{Cypc8S|4}GwOG;Jyt!gmPTDm?|$A1xUc-^_`(ja zyWzH8YD6;I+DF;9O-U#-M2KiJyBfmeB&}dA+!h8u>)r?}Xs&O>`6k`VUUw1ZTq~V7 z;orXT87E00TJU4oSPS7`lFE0yY)fAeQAJ|_mT8yp_I)laqi?bH&1N0|aSm~glxr3M_!rmcji z{5xetAk)cp#3^;!&c#5!O3+(lE`N$r?#b3A9ND-2D>yB2tvp)(0|$$^jyt7PpdUapxuzgc57m^ffN4Bwk+v^Lh%qU*0dvft!x(Ad$e zI~0tnAMsWp)nD;+-g<4F!!q#9v2xmvM#GpZWz(e?tb7Z7rkbFl%vE!q7D`(heY~lq zb83hTAEpS-I|MH~GbRXr4xA_bci#SNlW~QLu^V_7#u1QZPN9};G?faOEOTO2je&CdUh6;jA<2~nvqNBe}S_h`Y!qeiY zUwq&ETPcOdi~IY>(9(*8J4U&{eimQ4=E!~?()oytPa5aI82nOKO${& zr}rX{C_UBuydcs}DFX{ArvQr_h*5ng?7YPLd99md>tKjMyBp z`~qL*t@?A<0v~W(tpVMR_Pd-`zAbYHIW^_b>01bZb|$*f5SnjPc3 zes&Qu)vj@3sSl9URfK!w5pMlZkE>0y&EX)iCLQpPiSyDLsc!GOF&{}K*}Bu1?-sX= zaCe(!b{L6dJ@1EP$5Q^u#d=|>Lac4T;-6R*`;hq!~J`y0o{ck25 z$G8i=w*g`zg)UNXu0U5hdkefRzSo~TlXIBN=Z56fx$x{fJcGJsB=BSgtN8`p&F4M7 zNY2l

    z{A&8g*TmJBozx6|y+i112#ykqKSFr}8}75tNTBS&KiW5FkcQ*yEXQZEO~ zY0BmKkWn`coI6P?=u`wuOHd=kHd4Gmw0I}r!j^^z!_XG?Reh540^jnzBJ*(x-K20zDZWs zp_LFkAo->4llUjEwVIq{nU|pY-ah&r>I=?Q{ON03VIQo2d!V=9*Z9@F;`5tmuMYFS zu_L2BlKm^6PH;mRGa3k-$FE8r%qR=)Q zG2Hj5a}F)8EVubP20(#q2g`FOA9^ZxRE>1_*Ir< z9Qsnj*tLdnFM^3R@q*Lb_0bp)hvD|)@bO)?j1CWh`9>LVytrU5WH!g^`h131oWvJ< zQcZJGePoDYdbDKk?z|NkJg@(8PE%G{S#Xe_>XiMQY*r_&SgRa*5Vwf;!Ulpq>h%?6 zW%F*Gq#MZUX3n7Cj23!Q^o{LJW3a6fGVh7d^<(}7cEgod^m)GIZ2sZm?T-(d zS$RY@x;G?pX1Oj<^`#alu z{YxwRO-;O(#mJ#2l@r|49l9m#($)Gb)12*8I$YXIgT&Zj)Q*V=P=W2vJgqp{XAyoD z-S?Xm*8`2M#$7Mkn?`?a;|jqYpkle>MSIwP5(gx7#=qu69vE`81jjn1zo>_?nGRB? z(GM$k^@HVu9!Gd}2+(bMQsDnAwzp{sAzF1Yn&M0F4aT;$}tsYXAPhoaxdSViOOgre>PU!$yr;N~g>SFDlw2FsJGp1yIo1!=?9?x4p zsn7Hq*w3|&$G$>B!p>Qwn@Jp`95QtET-XsYQ}v0(XMJq5zd+O_H!N38LTJJhO~7ZTteCW-N5mgS=Y`=Z8lhBc zons{ZRr_L&lMHzWOe25Ic{L}PT6|@D;B4O(35X+a#j#!H_a%-Qo z`pbwVRICTcb2(J+TYRXUrYP3l95Wiq<`pcDrMP5X|HvJX^@&~nE4!DVNdm?DZe1{! z@ov|nVV%)_Nak2!Mu)v9r{19kttN|9ozpRd8JUUEqw8`G__!vE!(Cs(0<}RZOlQXb8Al$3Y9o72u!Pwbb zERacc`q6YBbi};^uS0*hU!p%uleWH)G`FJXnxst_t!4I5_rW{zp{MgnL7p}4N3M?j zhh?n^krg9U3#OJZ{Ib+SGWLm?quNgHq*K9_!uX`{c_lT@Hy?8#T*C;oeTAj$R?D_DA!$ zT?Z-O%o_6BC~rfnddNyd@Yoj;<(?VvNiG!=s9%eGNv$!qXfIx5Gzp~A{?r!HA@ymd z6YbD3=bm;a%p7XTLvuTQsNeZ|4)`$Gi%Cv=zicG9YM1?o&eG0zO8P?MHg4m4S^^60 zsFKOK^vtu`$tKYFgM^{ZxTD7LhP?>`7)S*bZI2sxbm}s&lq|RSsEx{(Ik{>%C`}&B z>xw{Y1$23gkNg4)qn&B#sU{t-s-mH|g7AbGTN@GbeG><75dbqd-7hIRZ>-#suK4gz zd6%TWo6IM49G&n4{Y(fHQn7vxdL`d4hXLV212x}#GO7Bhx3Gb8L zD4wtE(9+VDS_Mv4Q1f$}GFDtyEE27j2S`^_{zeA>c+c2g3@fiXqNOSHe5f7KNdY)GRn37!~)}ev18tk z#T+_;4FT}<_4F4~0g);G*p>#4>m|b!eWV_;GOgd~0&b8+=k2%i9GTj(cw*+8A7AvL zGUUhlDL=3`SkJlR=LNh|RPzW2{jJ8uePKg#ETRTFdvCUNw6CJT*Iui{{KnkiZw_9( zqi9T#+vJeAg57Ku-^lFt-g_RlZ6B++T+cMV3e718(RQdsuKCgMg6Z(b>Z=p+?~>m{ z!~+(h*W1iTlAz!~rn_oa_*|p)J0qHI&#lGq$@00UCiiP6Lb87s@t;y-=3=LgDs~X| zDUw}iyl(k{CO$Xo@^v2`kLfq5HH70d zC=TO;DgJtFfD$rGO%B=DpFnK$wv>hJ`#08vXE3)tXb|)(hUlDEp&hjmZFuYML2o-3 zyT1FYv~wL>Z4KhqD@$;pF+%wjf-DMbSQeQJIOR6P935@+bipC?ak=-3mlt7(r5q*M zeV0L!Y;tUj3K2;XEBT!+_$y9tpQdfH$JH+>J{=>7CCX(?UxygWzvHRmdYWo?MUZEa=^nO3N&aMrgs-YFs8^-&c3gRh?(0wRciTo$A-x-CtSW=U-^g_gmq2s zITKV6y2gA7W z8QfoC2K`cJWL0Gc=P|Cu^fvB6Dgq`c1NZvIP>E zry<=J(`iu{76w^@Tj+IzA>{_Qimq+w`%5=W#`0g8Jh1J-#FYxTY1Zw*7b@juqn@*a zE4e+>NaUM^&Ah)Oc}oWxI$1T&TWN_8S~bjgHKyW<2WOmqru75x1cgSsilq*|yzEV1 zBTOnurp%vqc;)4#1W8Ni)5Sfu?KZU$WaFyY*lHe|88lteXKOCc+sF_cp+j|LdE>#o zC^S}2y=uXG*a24FWA?Yb`u*!@bo-uV%d8Y)GglnpT&;cDP!&Ye{Nm1PkC9>MPAY?V zQq9OU-nCq~VnnF2)K!({Qo2#4GM3+qPHtUdKJY8`2%F7(%9wje_RqA%ROzpaEk>ia z`r_Mgsg74O9rTNsiuu2YG`V&HR?jO_=N60$@M^6c6Q)cw#p6Ct4b{+oEnCfI^g3&) zsPv8so|Rx^v=6$PwT4|SxOI!35?}f6F4OsO)Ts!yWp-@9x?Gfu=^I|J@zrf(xMtQ) zOxZ2-{_fV(OsXL)T@)RF^jA$!ag2>fQMa#zF0Hlsq8yTEwOxOz5_7bl($+8FMU1Le zmR<+?q)EfRp)Y;29oJYsX8{=hiZM)7a(gEZ2sF(8UxEL>Zd3Dv@kCriK%jr)-^HN5 zy^qT~S1Si+FG27B`W0|@wfpo+Ta|>6;a|N*qOPW-2LfT(gFqOg_*fti2z#HR1q8wZ zz0%TG27y2rScEwE#6;Mbgt$Z`_}GMmB>2Qsq~v78luXpb_>`m+kI0Fc$Z?n`N$BVp zKxC{Wj~>%7a#AopV_@T^=M>{01F_Q*b23s3F%vywe)NPD%+E&i^f8kJHzOwp#}htr z9wA9_ehvvCUI_^aMt%h*Ar&?OIUaGfr!udeODXZm8;Z+oi6|O~s+fqVTSzPGN~!6~ zYMP5_SxISH$-cCg)w7r92P-{eRuEuQ7w1%w5z&+7)qU|yU0%vaLBL2w%=)FIgWd~w zqZg{GYRcLs>Uw4d+A5~HYQ{zeaykymh7Kxbo_c0BnpS>h){aK@0S-pW?=AE_&DET& zP5s`z46rwJba1e7_Hp+3;NfiN>*?a}=d1NT#KKj=rN(2St) zPamSQoT3VS;)(*2%7W9Y!tAv|?exMN^}}6EV!X@}{ocg*+J*&vNC~nHfrZ9|1!aai zXGVW0jB+cE^(spWs?LPP#C(c~OAd?Aj7><1OUn2h`!O{>Ixit4H8D0hIr(!+W_o5$ zMru-aMrw9eW@yTnu(Y3HnH34?1@MgG_?+*NIW7A(=Sg0FXX&RVp z{Wa4%yx2LhUO9}YAK7S`*y#C@@vAO(sOf8ed+k(v>3B~?Z&&;8?z++bwwdmV#b5P^ z;rh*q_MxF)J-=rLrGSm=#5!UJF~5OWKG>K!*jhT?TRYzvz1m&e-P_wdINv`UW+|01H--%;y!#*hmQRE93Xk7h-Q}M6 z=hp(uZOpJ=> zcarAQGF_if)Y~OMM+E2EuH1DRk6US5s0dH~hp ze~Wx~9{oKYES6;Y>iErNJk(0Fv+((kCxNRl$0*8k4>DLdrgEJ$xnw)uxa`X*M&?ba}PVQ?(@1IZCF6 z>gjvUWU}6J&gYx)-`H(l%#5dQiY*2pWAFwvFDe^#I533g))Y%#p5E#^Xh?$d%HKsD zh4kQ4)>dv5DyDwBrF>kiH)K8Lg8%Q!TJ?0c1r1I(0VxNU^C0qq!b0#iz-Vm6JCC;?%>qNGH*7|MY z`3|tHRS(dyJhjZ<6K?Z)be3Hs7N#b772MN6ya!`W8jAB^zYqDWc9O%M&_E%Dm`}D8 zy^Eqzx`C(ymc;;>6)jtSR~L)OjWAy<=bs3){w_78$oglkA6|PISN8Nae-&ynpvNUz zpR~dB+gNvych?*2nev;PYU?u+PjdO{OM~YXXg}ZYHjqUmU&>m_qgzKIajpm(GlBhv zEN3i^Pbt>(VoBlsC8*HC;Z?%ZoJ)6U+`^Jv5w5d zLF1luYjTI5HcznP^84{9$;&?)yC)@XeC!mG39?!7fRH3L=K!(kn#v>I%G^TfcxS0m zqh6k+$pI*&YOm7Gj$}caO0f4Mx`ADa{%G#+F12T#>;XYd%9>6o7jaVZmQu!GWP=fj z`HMaD);8FfcXHlR17bewy>s>qZb6x1P!4@_hrHTTgimC15j=FFOPt@r7*|__wCqtV zO|^rg%W^j-5$vV##EwgcKkEq-dv&}wlj%SeJgz7ZP4vU)8G*D+0+RyH(h2h;Mjc42 z*EcbL8|bvu`D4aOj>FeCx8FV3-yDCQ@Vya)1QssNy<>`b62Jv+39Qt&D9k69a?Y-p zxH+`G{wvBjh=OqO?wmCK-Y~z&&3kJVQlQXT8mxheB(B;hb{D&N(xCkbA3s&+hCr?0 zc2ah-SaSYG#16;di*C>x>Hy=}BV74W1Gd>lOx+8T?EJ-Vy91`A8fjYr$` z&8j_CmtCdLD{7T~uH`ye3LjuodX8)O`s#Q0<*}o}8yV%@r0S0aE2E#3_Nk129wkf{ z@MpnM3X~JRDiV@ySx6VBJ;lvR&W(d|X%~9nd;U)&b@J=)(d0aESFr08;pWX_=nD@e z$k12#7w_IX4W!Dtd zoc7x6rU%2xXIXg8GvJ|Z1>G_m&{SH9Y_xlHoZ5XRwfbPRSzg!C;C|s_er=LIIjeji zNXzW9rr_&*R0)A$%&68!Eboj@bujTZh3Kqe3_c%O&aY;U*qR&ldk0lnQtpNtXf{xFm<2PDoeps_y+%-h(6VNgQi%DROMR3LDRcc+l`-}baZMp62Q8%%Y@VfG;9L92&7pp*QOtwwvOeYh zRlB9Cdib8kQmY%mL&fUVvR^;28)s6 zu6z}~FX>0Me7eHB(Z+6R@4g3G?!KNvzIy8Y4PfIC)OU?F)=F4;@v)@3L1Vg-Rs7KW zqDh3C^W>?{$07a)v7v|(20fLvvV>oeS8TJRJk}C3{=vTsyjyBU=$5=))T9oDm~{WL zRqWXoCIstKDg+9hKwD765aYNa9rGDFcjmV~GD;no?o+J5Higs+E!ZMeq?VVu83@e6 z7^&b*+O@|@8g*&7%oH%Q#tbDd@B@`hgr5!|$Rsn0Zwe;VHTQHC^~NHxrKk-u$dOtgxm?G=$}Fhcg<-ine^mMH_dOku%U55&9BEFy!h<+bP&&y@QR6g= zlIwv_A^25vu>?cCmPw9y46~t&+5(u9_jEq-`Jq$xp8Mv9ySkD?;u2~9Cz`kuJNbls z!oipm6dhDHYc|Ip)$2-2$)&^?))Qo+i`lcoDJ(wril2FW|MT%n`AM{)o9OI@Y9B=6L2^w`73nViX2h4iTD1na0h)`TYv5ijDYo4SpmWUIl7m!705 zmw6a9SbC2pLlcq16@~gE{?ojp-X9i3>%ewVEg=Uyngy>v$^818s-A!XHx_W~e+tTW zRXgQE4LRE>h%-RCtz0xsNpryxmN)3=fp6B<8N@!@u!FeSf|XheAzcbC8RDT>&>TsG zPxxL9J7S8b*WaE9#`t*mqqqZK$;8h^+t2TPJtG?J9^gY z``b(K61wtF$N9pz#@19e$aqUbja~mTk+BR$cYM=G;#!8ingH!+AAAe%6VeQbXcfc0 z*6T;)q^0(@9~_m`_Jbs#^Kpw9ss$bs(GdeL+dJU`CWfay(>m)P=lC;!U(~?5-^(0I zr!^BNlEP$sXz_vCz`!2!E?5{9Y8u)|5A-l!f&kn%-H<7thL7(N>%VZ2svT{v zr?uY_EIyiJdzbs10^%enE`B=%Of(FLt4;XS4EW#dkciBMwn z>IIU_)>^N+7i%$4{}k5J$&2jVAVmubpR#PqosREHxy6UVq+4}$#yDh2Jp=oe+d_ai z)w7b>?hy=E1Tgp%X&GE8ja(n=g5+U5xX|Sz@<06KxW+%q$MBVY^$wXtcAD?V%a;6e z!&^Srs)2M57rk5*WfHHce+toy>N^Qjl!8ccYa2-*2945o|)aY35((hi(r>nX@!ZCul z`hAQ4#Qg9EtmgMX(DBMD=3!Gj8uBN1q+fj%>?d6r3vCm=EiZ?YvRwl84zJdEk!}Xv zottI|OKFSl^ey1Rc&M#+4_Rr{-T9=KuQ?31dgkU41{}x`{*XeS1rna^_<5lk*K3W@ z7KksTBq*-d5I_VMosXZ)_%eH20YZCfxt8}lNimlm**JaVS1HQz(gy{A{4jt{RdL`Y z%`-M+<(?<%CY?+6aP*92NEA7kH`QE05M0wHf&MHwo(*Lcy>wi_S) z5+5z^gW?rHK#Z|~6;T?Lc-FhTfYCbq?#{Ruf!bX_2@4`UVr>cx6e2c}kaL9C(k&i2 z%J&Yw)%it-^nCKG8k7krtiduw)Fq}|1k#}L3F~;YPu-ecbjNN%`h*hQ24Q$TXsKR* znT^GV1SoZTR+iAWzIq0m z8$CO*MgU9S`Pk8oct~0}P$`V9uU8`PNpf{Sxi2JJ)8MIhQ*h^X+s{;CtR;ay(F)-D z_~_{Rktu}&O#CRutY`HIft<&RL6Vzd0jbqVe{Aj5Yn0U4RNXMp3HOOxEc&OGq{{M1 zHq1{hu+hGmf}#w*dHlS!&0(-dSHcg)NEHpT=EhQ9at8V)}X@LM_azr4aE`RAodHXr`@xZ_&0}}Xrj7&gC@0SKojJ3_A z+hmAuX6hS~h0&fufcTVG%_&;Y4^O`>evw2QoWlH~anVBD9n#31nX?20tegk@T+I+_ z6ZV@1S(@B!&{Zi+>E@I0%yjDLIh4be-fd*7_W-Mfc2Xw zlaal*-J2BVp}@c?-9b~LSJmX|8N?<-)NhZ!2#S<0%FT83!~i<91;|exdsFkF8Nbjc@0*P_Vd#X7V`coy1tqQnexF(J6d&5?(mVm zAv$sP@+V4E3PgpS>9Xv2quqrQ$Ih1GS5BoEZIEBmNZ%6~7G22CT0Y&gONlF>*%mdQ zko>!X9%zia+n7AFlvH|NI?`r~(qRgy@K?Z;?ODU~XA|>A(OM6{vW|W@WK*G>-~e>B z8{kHJyY8j0nK=B8j4PxS+JcF5(F5Z-7~K2iwy4h6hy7=jDi39F*di6mE8`62LINJB zo$B%~wg7R(#b_IP00VgKt2qdBkoze<2uVP|-brP@sEdPXGXY>M z{%O6}T=P#e&U9=?pOwR5igSLkxM(s(vv(+dCO{?((0eW=>VtZhcIGC5Ie)y-b9Ula z%L;+OYDrAa$!-iaay2D%fj9~1jt z>bKDfANoVPbLHJ@G#GgEE2*o*_gC~Ig;>sye>?p{U=etQtD2Yl&gw{gLbI9mB zhkTlzuFfRdu-Gv@^`s8%0Z1hR)pq6@^4C%&_&k(X897J%~S=&0`p~PL73`Jbp702ZaD~%^s?ZlqIWgV(wd19+0F+KzzY+wB{2u4grZK zJMDZYL5lr^Cg6{I3!}$-l%;6!A;r~}FaqKsd2}%FY@!$%0NKjlH@ELvY{pT&l*ySj z9_VZ=W3j{pkkKh9?B>pAEC~Eg0k~VTUX1i(X)(^BZO%$$&yPL(A)+i+rVj z8m@Q#K;X!HVJm_jKPzJoT=5n>J}B3Pyb;r#XqC26A}UmF<&S7+)f%s;txXV1D0*&= z|8pZS%1i+6JJ{LF1qOofS1TmZ`NEoM?GrgRpNnS7SiBO7pc1&iC{J1$#ZJ?wyU8v< z!1Gsi-b@3Xuc3Zb8B1&V7BHs^SqTIFB<8pmN;UWg~?tilBP~!ycV~Du$*s}a};`dK#nC;9%R*T)j%UDQI zl?hhC*)5-9%ktr&R>Y`E)=S5QK>iJ(xsnO5DxoglVyMXJI`rmki~8$r3!4FjEv?<) z(@q>Pa6L|V-bg)xqkB=CnSJWFpK+q{P=_ z4KAlxNI%XPQhO$Wm-=;6k#8gHU;K-hSnu}!>Ycrk!u&EMM=TfV>&af{+i)Vg1rXJ5 zy}(3wB6#*y@c{q#LkbBmG^|Z#bi!2!U@31_=4TDh&^{$2Mp?h; zRbc@E-XegQKJW281_Y5GL;JY0Z?l6p7Y=K0I`K^3>~IM1SU&~+GNGOb24myJ{#KqL zlR}s8+2(a_Sa_p&tJn%BJc$M&O(8IAMCe*1?R>4CEB&r{wxWM2A%1jvj| z*MByhfN;8D#b-~@-iF1{Lwq`&&j;RXSOP^*{nLDfQk*}>Mlvz?`v?{=@TMD3fp)`+ zA%E9yhgKy1QR^_QPBg%tUkV-NM|~rU&HsrR4&2Jn-pqmloy6Nrbi()kU~&1AaG;a| zWf^r>pP7B;rxSbUF^+UQ=RQ8bLgX_WgUu2n+8^mceChP{yqj%;oAF$4=w3{bcm7a* zIQR_Yj2xo>;DoB5BvSByd~`%gareO;<;77Kd1AH-1}y2XrneapotdhGorGw{WCeDf zZ7e$v%M*HlCyhJcv|kp<6vl#I~2o|7Hy4C5{7zc~MPZ8ulXPRb;NT&2tcp4kjjAa%?Y z0taL6aa*L&PVJ$vzsPix#20R(G$`wM@<6V+sJ{fLC^x}oTaWcAfm4=mVL-~Bw_UF+ zve83tWu62TwV2Z%+$xEV*UUk{e7jIlB7-o{JOV=14U`8_-^?-r+NbNNAEqnwvuBk} zDi6<2E0rI93`8{dPk zbd&#^h(5FuJ@|Zm^G0XflNjxjT3PlLytKclUb+lgQ^^igY~yJRmQ=AJ85JI?XUYvJ$ox0d@*c8}i>pid$}C zqiX9FJsWD9P6WgUVcD%)FkhnqNFyAW=|7wZIVNTl+JZ1jc%athDlafZZaOpfIJVZO z&pH!fg8<93{x7#=4a@y!f(29JDr%&@<|R!J`r}9=7K5PeY?@qFl}oTlQN%8BX4Hu^4s7G@On^|KTF)wpeB?0nJ;iY?%R=r@wZ)ZjxH?I0u!kiiC0Z|Syx z*MF)HF71OPO8BhCGiSgkH#)*eVpRT#ZK39yo~JpL6J$2Od#nm%URM}!Cy*+ii`cUrKli#>QY$JyB(-iYQcz`V@|CZ(P5n5bjs(tv* z59uNU9wdEqSPA}J69q0RO?!_E>S=n7&@_;0iG7egH+4Mna3rO>r= z?pSCD@2^tX7cpK+vl#Zl7bYuA%~nx|e(i`4!${g6#e+Xvef1+y->)nUzcv4m)%?R6 z6JWH#c`(>7{nQsGCxJeB_MdWcjC^ThV;c7R`D~UtnIc(P(WoKN} zCXK#|158P6RR&>Fj1&j_>3*^QQl&oyxppJN&WS1S%Y`2Z&lP6lU^J-4%Y!#7={kd* zKc_NYrWd6?+*y9S3YUzz^9Tc`O=olx_mM2oa6oR60NqM^^Jn%9vICX!&e#G7;QzP{ z@Zk1&bYqb5&&|1fM5*9j+INjXOTNw!SRiQS8IyS%NNPz|0190EeNS zt-fyo`oF$NT>w>^9axfBfTTN>5iJi{palBcL@qvh(31e&eu&70!@Pbzy6OcZOPv~A zbs^#_U?6=p=H7E3>07V`8%DOOuox6WC9r_mh>HGT77vvN;&u*Xqx&Ip{cd0hamd+G z1oa+-h0m9)P)Qi zogi@Z?;@pIXl`c7J4w-3n;oQA>M?G=__}~x(SH$P6{_HS9dxquWYo6^+D3arqYGKj zA^KgTL{c6(g|Ggc>|QV7ZZhT`?R7ftF{6`bk`$J9N^?a;7N{_ulA2qnx}aIGTT!vu zQMluCdh+?~1gk&zfA{y~`HvX|y^_wx+xqK{bc>;;=Ecyva$QKkK&Lcvr=jWOAu**F zj;r@+H{={Nw}g#MjL?J!KJOc#=Hfr)k#W9x@GT?O;) z+rgrVkmAfQSN`JuqZq$YBoKz?eQwV3>-7v7(P$|++8 zfhd^%y#;7deelKukU1Lb+{b~e(*d}~GvikIAsUz{n3)sXPiSRSg)3tH$(AMA_>RT< z#w2XaMF&a`$uGG3iW zag{M~Wm8+%8lkjEZhfM;~ip)OTq2r7sI{q*`$wkIPXL&(9WPwY>pUq=@+d9IUC z8(^~AaC*s@8tnP@s@=er=8yaH)ApK}=gpIK#sb&nksj6+^HFY5Z4>8n+k3B=s>sfJ zwi|w3j#*={^z>QLGCr=ooo@bkSXapi&jz{O7=fFgl*V~BUbc(bbPB6`s4pz9I~>R z*1CID2|FM9zWVelD_48{nnkBUTbdY*wO*9Vcs`sHHMs?x9zT-`q>V7x+)EiKa7;rb zr(9s`P0#k;%Dh^n;yxi5g4rjN?1!^tr}0`j zJ;q+vq#(GEVJ>zTU74VfFbi{UA;Q_fTZ45*!CzMlA#XiwL1iWk`aoLmhxac6`-0c7 zu~`R<$0r|fzp-j`SJHm@n51DPMXvwTBcgCvm!VzOKBV*>pKSDn7HN z+-!cgi{fz9hJ1W`pp^b?#XB(u5R%@8c%TiPF+q|5zb@ue6D|+ibPgL`HH;)EFC(e^vrY0d^d2fdD9r9yg(^adEc%M04 z%@I15vfSODoYEU-WoBw>R4pm9eA)Ec$S6u8g4A8iq4xNjjd@pBe;>lXw3taG{hFEg zL#2!~J_my|MX%|o;^mbakR5SW!iVqKr6U;;1^gPg5 zf=lWsF`risTgf7rCOlEfDQ|lzKATZHb#C54wtukCMH5pFKbQSF+WHIM^>X1keS>R{ zA%;VShHRI=ol_94+W-D@+Pq)b(M}BRc?@~2=~Ik_?wFyE40(LFTRoSx5a<_#5ufB! z9D|&T-cT&!ocuUJaR)<@MMzU`n#LcM-)fIN)HX^+;-WXSUMbVBo-}GurOVAld{sd| zMHXfir*R+1UdU|!BD0*Yg*>#H6aW-hPbf*SM@MB+I3EV}P;B^ec-B7cht;m=)!JbV zAAM*XeS8KjPiMZVG`)roR*{~Aup{e;&;(`r_~oZWu)x_9e@cOvW!DMF^PX@boFHPj z&5kuS%tDM$%eLq&UoA6&K8wE*vr*O&3*acwV0&8a(j3*WCnPeX#8|02Kyx!-UnP?~ zm>_>%=8vST3|;4Yw~e43&gbS}qsX8-A@C#-8ahruBc7$wo2-w#f)4o86zks$p~74i`md}BT16G%ZVcOY?wvP0L{(o4|GIb zSpRs9z#)TZ9D||+qlvNOLg4C$BtEL_=aX}ttX-tNio?W_Y3a$_AUD77|LJB7!L`$H zn$3;0hh{)l)HkjbvX$jsXAoZTQA-WJRNm^fpd9UfX%odK&U^G5hrr-Pr@;pUv`)r# zgAVY{!PVA?0m}}++D0d_NA-zCC3ZO$h;m%QUsP%rW3S;^enJ7NOLJ+dlq6mvdXF>_ zEcM6sbK{UfFHo`tMo>1ol%!mk5B4v#*uvv(Yh*z{5j2bwW!1ay>I7^n$ z1mVsWx~jGQRf5#4Y|YJm&zL6{6W#4;!{@y5hw5|!c-9)Vk2BI(lQSucT&u)t)Z#TB znpZ&;2R(-knzE0!e05@swJYuOTe&P%DwM>#N;I0NZLV875pntqGq|oggJ^W!A4x!O zbqm~HY2Edu^0o)ABeFbD(zfeoZMlyA{mp=XYC~YI=RUf-wWk$V{qapFxpj3`I>w2h z?Zi%#D#<&0TUMB@JgsK9o0W&AD&LdHN{4KidE(a>=Gg{A>9mUdkYq{2VU!d;#gr_9!|0PnvtCl>SA{bFdn3UgK5DV_3|m14o@bBa+RIcvYk9h%Zp70pLD#DvjFpzMT7^Kr4vgN` zrHk2ch(&uSd@`ytEU%B*n^R3d3ml7Ui0us1(OVLfWTQhK?|K;vwnoVafUl4N`6zed z5j~R`&8Gh2B1fNwfQaN6549N${TW=&)E>GBR-Qf_P=t~_Ce38Q)!!d!)(i|8QjT>U zsM?fmJ{_83;b~Lp-vIf}1poE=VgCr~sIle+b&%&D` zYvG__wrt!8xP#6bkVVR-)GPb}ZNLi}X>w{99=P1!rh(yX~X4VV@b06QxN zD4=f;-DOv|3|rw!BjpfB6DIfha{69%kG-q2epjT(CEP%tFbI#5tdEq|T3N$B7`W&p zy0TUap^S9MQP)nb`g_x)(7Wike#@1L0R-o@-0%!$s`ukFsbzC|5!%Tqbxu+-n3-WWW3QH2=1nSu0XkKWV!OLo_2Q{ zt`?nrAsDm4vEZHaL>hjEkF`bZmhgwggQWJres9f9ek=j4&P*5FGvf>p;K^^UtDTo7 z3-)Mu+FTKj({wnlG4y41rf2me*=oUPqE;4c4*Am{O1{nCNA`Wj7nATd{B%r%?)e#yl_ZeC~5CF^Pjbk=4 zIaW@R$9pN|u+1e1-JM_zZyj(LhQVD0UE!0YdA7yXaCOxIXX%Zd3mtnfKHXWqofO)b-RItL|NDt?-8MJM8G9)HQKk=jdpL^}Z;R2By+ zmd61sV33f(pzs^f(V|<&h1Rl^$30jMEq=&F$u>Fet=vCX>-OeW$k`3P7X-9*GiI2A zS(1xuhMuLPnxY%kUF^-(uTHEElBtF(a$8LmzG^NU7ez5h zXq^G{ro%lH36YZ?vOIp1E5puM!;&{?M7&ZHz`sD!QahTWt>E^Vl{Pkor7)5wCYmsM z*!U7+)?%TEFii?Rmwn%t3G9HM&=)o&7_VIjU+0~&@uK+$11%C%Ao8B;W~Asb@95{B zGP%@3z+xH=s@@oW`0;%Ittq_19Bg))y0&`>uGWQ9)F@(WK4b*sj z?i+^IIw|zUcqxp`7eA#dms@9$>WayA#T_Ni0V?%n_ZrQwUcC{vHWTOSw36LVNc7TliD!U&OVtt z89sL~A$HkG989Y92@7P6hwOi-#)(8AxnU{Hu+&!3N|*HAl*EkgyDy~f%?^+wi-;U< zXsIfqm2cPm$yx*J2c!C%y>m=?I_K_B%qwiIJdC}+#;95_h4;)Ku3l~k22bp62E82K zRHT$WhUiSY!f95yRk5IiTYDfy*K)=rEdrHKti&mZ?2*pSx(piLp!5Euk z$D6({99>t@x2{PUKo*mypc>3zxZx4|kBDy`tn~4#iE9OKpgq=Kc{+$+GDSg;@h4b_f@rw*o5S&tBp> zZA?dW5m^W8L%=!6bsr<=8rGU8hdqsToK|W;%BxJ|FQz>+b^ZpATKrLm;l*4s>} z9O;M}>w#JLn0mf`XUju_X4}(a4Z@`!7A!|P1hgE^+I zoJca$!d&Q#BQ&ovZ0HjeL++(O&-YS4pPGYoiyx&%DCbmHwVKbeO7gEwc2D6^+|9t9 zuywXcTvJ~t_!^0l^`eocKxuL_=EDJIkYo|HU%Raw<9Vg}A~U%Bx(bU$-B%TBEhVzU zuE%WDouxZIw16N!9HvRVVJ_u-6ehgMI;{UxjPH@esK?>C7Xck&OYuA(-6br{IG$tr zu7}?QJKa7|LD}ne9vt+Aa!1SVoXyox8qHC0d}_Y9M>qrBq~gX`c@z={L`VeILjr{(yW637{^uOT&uRhz4u(dG!PuQEI zxwRZ9{iP%<(WVFz9QV zT#(i=(31Txs?X<+>+_@wZmmc5Ypy9jY_E~eTX=TSv!W2ood?Nw@JLt+wv9|2=drS##W4jWur-=KVoa;05kp^(>DibJWMG!Y&}S zY&`r^8G9 z(XuCNz@&s(TDp*vGk%8eP%hp&JBhx{KoiUrg>IZc5-&sFzwKYz*EtB7HAh)ra zzy9`i7Q=V2=C-1DaQT7R+}URfw7IR8-}w9wc5`R5Uk?rZjoCJa^Z)9i!1%x1(QPc} z&MsTz(QUc>#^q;B=gvM`*vxGqf9LZHwsU8vU-t<28>gQ!o;&;ex<@=r|9gIZ#d`kj z^n*I?liIeje)l^#D!zex2a|k-5{*=e7)lcOC)#;qv1&{aih^>hiWI bMmtpLD|)+Fw>INq;9{_6VPLSox|Q@FkEBFS literal 18269 zcmb7s1yEhtvi2dkdvHi_ch>}W2_D?t;ot;!3GVI?JV4+8!QK7f?(WWCx%1}D%)2x9 zf3K@f)!D0df2;Red&%zA{VB>oLSX>_Z~%Zeq>^foHBUGr008*)g988nHWoG}&hGXm z2KM&W7Dfin7IwBwKwD!*I|C;RCq_Ga6I)|DBNrPJTW3Zmdq)!kV<&SH6KBQ0U;_Rm zxBvh^#Lm{#!pz0-UuaIOOwM+8)`kX-O#e46Q)@c|XOsV{mW_$6%YUv5|8Fd`x3hP# z|E2mLY$W^}Jp&^n6KfMk182McWar=2_^VCq4Qx%U|Bp4=JKC8!nm9TA-=ERR*}&Q5 z|D#9pH_vEcV_;_D#3XFtY-3>W^k3$}!NL9Ob^24!1CV~bP6oCXrY26#jE=^pBk?1) z0nBKDr~E-Cqw^n_prIY92b(Oj+c|2R-?bC9~%(% zN7R;dsIVK+Pz*|%*YDHuf*6|$7kL-v8|X;Vh|Vno3bSdOU2SQ3^pb)YxR9*J>hh@F z(k}TGT1Y!)xE&;sdcQ}*!4pRbTU>8>;q#1cz?-ToIUnYHb-#Tl5V6c#np2Bl(D11> zq~Tqx--jr8*}*{P0K#T@nI5*JOC_F;+~bWt{nn{~JbwK%y6Ca_l~AOjZQZrj>J#mN zRj)*Ij}s#gm#LofRk8&A4Av8c0}!A4>+TEV*el*f)pww%^@q`Xvxl#{-!Dj1C8`u< zVBx;snomMQ0004y0Kji!;P0C6U!&2)*?`f_#yU!Q$aaw#t>u;uic>wDNh}Fo6T*Yq zLvrGci|-0j>Qafd#bi!N|MTOZ$Gp^`h~3h}@aX6#dJ=iq^p+3iQLY5tpTaTh2dy^K zkUbWBx}iNvRuW(C!PH+D%1;1{8<5?+cm2r`2kiHp2MIyn--eXOdf?_NB(mZG&@7AMH_5AkEvC z$p~00EtXdNbTaYdziRH6eg{Go!S8EJfHe_C-aU9O96s0-YAV%g2?4h?s zGbDqlN9+e?-sisab^8>eB(}keyUh@d2~s~>v7_;CyoQMwMn%0$aa*dxiejnudswq; zmSds8h0NkmG+l1|*&|j1_OK~uF5&^{x@YN2$?C#nk`dA;-AOq`UlaTSs<46*-&@s| zM_rhx&r>oLV#!+{4dK3dV`Zkw=msipCsGZOQiTnMHbl{V2jS&HW34>H(6ODIyOdU? zR<5B;#Gc6f zbf{_p^sz8O96H9H<}gX^5XM#69irQ|0C#NTHm>B=9koOuQjGQflP{0mds_OO$S!1kja?_N7z^hR zu``%uUyQ9dv!w=<@xC8?jEd*Dhm|Ec9(4Y5UtTe_e-hVMB0ibU_86Tv9GcoY>K0H# zdK&i8P=WEw7GmYHR-<8RjOquIPkchC$is4ikameAuT%fQf0Dwx z`RKJU8vAXiYV7Z(dYY=%qH3rJ6{_7b?iHY>P4xYFO5*F<@I-BT zjA%-nLKJzpTO_ZhMw)tL(w$64kkSgRPkafJQQ>y^z)Ot@1^E^xm;$dQ+?ZWXcg-zBDaCl;@g*`_x!GgCfn*zOJWS7%tADdj0c#pRt>>*m5(F- z#N@;>^05l%o-D_=8$3_CviAa#{12^ep{pc?L@~4-VF<|;c$wr{1gfVhBNN~%246kL zqN3@3Y)?uOxR`TD9tMc8267BpY?&|*_V)uEa#Dr#s$9{=H+A1yeU<&E)_OGaULv#I zqk`+EW4=JGe4-t+t5e<-*L(yKp$?Vecb(sGNLSU3kUkwA$$vIu>{i&Tw7f4_Q9@3z z$ipVv1w|jhZTTv@gaAB0N}v(Rt-RAn2 zvzx_UK%C`X$r>PA6!YT$TtU1uVYdd}%=bFKf)N&=bN9R?(cdm5)1J$>Gn+Y$RJwUm zU5jj)%a#=)Hrf%0-`Ip{)q9_CA~bXLyiAmitB<_aeLrKnNJC?2M~W&Li|Iv-r&}>r zJW?5Rb`Rk+wJ*p-<*h5&I9BTOM0D?acSZ4Vp2*F8BCxJeka+OrajB?YWD|4O8^?5H zg-DWQMPXw!g#EEBgTc3**%1mSicpLdyr=s;@4k(8XB|-T8YKJtB9)b;^ig?0LW8`a zPZ&J#Xm|@5sWY?{^zp?!MALf$3?`B{jhHE?;9MO?Xm^w%qgGD1A+5W+IY>gtZCj

    &G9c}__7RMF>WuUzv_sK1t&V~7!XIv4<;>fL|ZJRkx9 z03$nF=U@ADHyi8a3>&-6ALxCnx5{uQx#l>D6tN!8@r2{8Ae`e9?7bp$zb?a%GT#|8 z$Pq}1sHjgsB&EfYwTXpppKRwM+QWKM*R!``$PkQ9M;c=+@&wlWd&xmMnpRGQ>-6MG z8OL_&uT8HXZ`D^{)Ux1)QMOw8M-XmkOD5!NDTQTvo4 zG(`k&Z6%}YdR?7P>S!J-Dnk;4fgoPyBs-){XWj}0d%5sdhgUn1@B2tXtob!7I7$iH zwJ=5SQLX6jgReqxvtQ=j_=z<n0i7{CjD{@43D$7y zD7;}nkKy9NclSFyA6%_t*oh}B@m@V&Pek0>X*O7>MKnVRsz|?N#h4LA6&qo3Q$bsO z%dh4bjS>5lCYI}mN#Fgc>$BGJQ4p*OGZ>mK3Y4FV>Zz$`I~XhF)V8n)9{u$y=oZnUO~VV`g{Hio}n(F3IRNn~d;I4|Uy25mQBJ{hA!OScJ1~&8e%~(oP-KoPI3^ zJH#F+XOxWwJ#ii9N_v%)U64Q%u2x&dEizt`F#uDbuRAoj)$>*M#qV>B=1vx zYxSlXly`{mHwkXpy=A$2qJ^Q)*SgfLI;|1+E3~QOf;o}5J$86e7Wh;OjpahQ8RI{Z zkIEwGDsVo}A9`k-Z*RkAWf@9)zI7iKZevF+36*J%8j~aoNwHX6hfT;8xW${}WKKZ8 z6~&YP646WoPqwXMZufewaNM=1guT>BYb(KmInW zyF=o$DHUzeV95`{xz4cD(5}W+wvpgme3PGj&9Kvb;hT3zX=vjUPFBM-Qhy5TpmuQD z?=D5)#$fXlx&uAS?r>J$?N@&cI$?LXIULC_lFQ)U#a72|)=%TUt?vd~jXAGlIrPly zGW1*IMC$DimZf#~Kt|ufQgY^R-?v;QW0TO#N{wz{QFzRyV@uFirRNScr$^8T)vpVW zO%NX@_PM%+M8gvxm9ga<6yWx-D~u~w6h~8zUO&Z(DM)pbQ3Q=PV9t`n6pCb!%gIC| zoPW@5!^0dNQ-ow3)b=i;vX_Y~N!}`l$s{3Hp49_%0sY|<+cE+2tWVKW?k!U^riO>?i-$ZrjH9Fi zAGI>hFb`2E5>6Wi#f?MXoVo`J@Af^uQFf4THFgP)rxq0x&7P8&(P*0uo{{~6_NJn@ zNv1%OLf|JatR6Fi0;kLL1xqbwp|)K#de>9O8}_$KVjfNJ8Eqj%n&&7CJE0915WAh< zhVGa3ZgG7>gy@lT9_NbTO6X$J!5}+98NtMpj@B?!Xp*%2wzg2d0T;qI=Nb?6Juu2nKx_XEQdw^dryP@(%buLTS1 ziekBWD3)4Q6*X&Sy)UpNQQ5PrPiM&Xf`bYvO~IyXP+5@gs^6^E7Gh<)>%clF263BV zIE>bI^GvetyhABNHv@4ZX|F8`!tv`%VQ`%_qik4XZ!Af5Hw|IzN)M$3aBIc>X!nb@ z5*~A;9}BGQzHY4y;Fbvc9i^6a=iR6h*=UU{39lb?F7(oeq#u5BE@->7Xt0!-o1YI& zLLo1H-J?o2TK|U5e)xeCPx7~V_^~&>Np))uVVFt%sz)J~yYD;UBBIw=y*AiLdv%}J z?NZeZQu(B`;##-`qRErCF+G4wFSfDW<;@}takWh!Y;F55YkqtP;^wC5>M&s=Oh{sZ zb@0P?$byy~3#R-o*LQBL-~d^5_->Rn@oRd*)>^F;iZmq;w@ln7W>GKU8DU>TOS^+k zU!rN{sSCxhChagdelnxgI<2nIWVkbKR)FWE_q6Vfpyf~}2dZYcD$Lk95hBd^h4*S~g4d1< z#c*i?UVB03TaAWlXmUg?{jAQULJhMseq5+=E^p;&6m89&2VFfXr{@^%@7cXPB1(lX zfDEbqF(v6QWL+saN@~mW&EoB<^y+Rt`kv$0Hu|Y3UttzQ zcuvwLn06RWE49N?f^vZRkTA9#tV7CW%%bBVwF4Mk_0Eh>``wL2%ySw{-5E^k_Zz@E z=3))L^W}H;kW-v~$4yzzS--VY59a)|NP%f(y>2iMv94M#{V@M)R ze`?0<662-mB%{s_Y1n>v1w%#jb2xGR1>0`A)X?=c>g(9HEBfHr^XoQ3@QYYs1I?l0 za}m$r_k*g@jO9z3$8){g)j9vbiDmJ8Tq_vwJn@uw(orRn$mcUg7jjg<^lBeEX-St|a`(_^BQyw!0g32@3zPhgDVu?&Rd5oRZ!Wk)qwX|=R9 z+Z!|yF=y$%xf0EyeefA>5CRJS+zJ#ze{(N_Q}pOD4}JIkJU1VA=DQ`A zd0NzG6|}JzUR)oz z;G0qZ&wT5mm&NpQA8wDP0k{1^cn`qPX8*4oLLWM9aTPIS;a%`z{6hHc^b$Pd?P5tT z-9MpxZb$|BmBnZm0a^kdNA+Ugt-JA@W%h{WDo^eZefuPO0O=5uE(|Oo>cl;dhO@@l zupp#(O(X(X0%^L){djcmB8B?sYi08`u-e#OKb;Sg z7k{Z1zRCtsG!I?-4IkrVJs|Wg^-$A2FV|ZE`S^K>O>Q|m=zF#yqisSKa=%AWW=P3; zNdK?U{+;0zS%?(%Pn1`#!c+*nEsj;i5VEv-@SWbGQSgv)3koqL0`M;>)nz&5@b^&l zGHCFQ`=q)V7r-iG>@Sy}>Ffv56m8x241z>O;0RYD)bq&Y$oZ#Ke0^l7-fte_l=Nna zp{cI>cngKcC7g?_2;>w5tbuDn;=45^iy{4wlFD%J@Ra~2_Sq_ZieDivc}$jlOz^QJ zd72RT7K`rG2do=Wvlmy05vqcdHf^;(o4>@iUq&&>WFqOVG^<3p(_x?;31Zh>{aoJV zXQI1=rZy&7t(!x7d>tv94nt$xWnwpom(j zqRmNkbtl8Aa#>yHdCua72H$)qXNQRHB=bL8t8Bmcp4F*8%!I&689ensEbunG;$N=>g)LyQ1^CwS`Qc7Kd2T&0*Go1#aUrUk384HC#DZGMRr$L~o=i zCd1ar)HB)I@$yG&@~cj^=2XhQ!b`=9jlA606^Ynp`aQGJ!`YB&0Yi`Hv6wUC&~5Dx zuco%d$D+}nbF;EZstS&YANAZ+wry}+4Vi^-hC{TZj)!D1B7sYO@1vE*9~v)ki-h%T_7aDE+M4d7j5o zy5m-F<3Tm*khZrCGK!6^XEGr%_q z$YL)aE&;Y;>(%)zG-_N6Xd@lw>1k>l$St8!2jh+MV68j{R99&5CQ-YBXE!a0N_!QT>}6Bm@3Muh`o991`!bv z6BCnwfPjpQjE;_ujg5_uk55=wSVl%hSy@?2OUuZ}$jZvf$;rvn(=#9-AR;0nK0ZD@ zJv~1^zr4J>uC5LQ0(EtD4Gj%VO-(H>E^cma9v&WEUS7VuyZ``z*Vk9TMRXnjK=CXk z`ccJg;aDduX_K=u@rU`UYuMuJ>o(+vrsk#nQF13zj|-I4ABssja(m$fj1bHYHBSxX z=>Bk{ue97&ZON& zaZ@@2*NfJme&!#xQ>oK8|D_K?E9{UNIjZm zO731jt);|G5Ovve&rnRQP}cbi5&OJZBkUd0f%byXMQ9VE{o%1TL$(R>qjX!#&H%X_ zGUGiFfVy0PKRk}*6}qg~!xA2oR1{~X$95pshNXJq3qs~lZUy)lCiw@fvlFDM#Nz6v zZ}aFmC@TOdyiG;qLvbdOaS0t(*75ubD-Q#q>HzLLtd~Joc2M>U#tvj!4+YE( zQfhl#c-~bqH)NV_+fa=nMlNX}7~3kpbc3@fmuJC(c5J#Q9Doe8BCZ-u$v|~y;hO~m z2yE07$pz$t);0Ab(Iw=Jkt9`#x^*arca*vjg*>mY=Wy5bcyni+?*q8QiS88 zeCC>Mt)+R4<#eez`9M~l=z{n5Nr9?URLwJQ1WbyH=M;y~OV5pHmKYV*xl-@Y-_yvz zulk0Atf$!VoP}Y$`D_e9GD&5|T0GQW zEE*-nvNLdQ$z#hn8K{#D4XcHM0f8gN(E+)0I~?n%qDqOdO9u^kRld!A=$z%{ zS&MK=gufiM0ueQ`euiL3Sfgc;aeh+(@x!#_l`(1iE@sz1%`Ww5ciC|*=J;yUGuw*C z>Qj4)JJP)4EV*8J@+amOMt~~m+RFVWW}vf_XMEfrs#N8BowRcN_7WmF@% zf<>*)t72+H?BpmI4Wj+eUvvXEgai}P`PguOvN@>Ab}l9Er?K&fUA|vXL(hPdhobVw z>eKgc`kf8XZafNbArZH?@`qmobtb(CeQaRKibhW!GVje28yhA89 zHbi{zX@qF`ZVFogX(jsXGp{P-ymI3oyq->aNh?Wn+p}j_YZeMaAo|uKkk$tu{%u~vYUC^oc9aMg~p8)jf zlAoJFwfrF35*0C8=j{q(@p?RQ;EeS|YhaVbgbqx6ciG}nrq`6*ItrT0HV{}+1|$$t z0rT^2&?~btJXGGSfy&C-m76MSzMasEy1yr&kap9SV5D8r7!*TBwHgvdwlN+NHTsH$ z$qfy(T`g+=GRmr$n~(BV1}cLB44=yv+SgR5(&CC&U&ND!E{p~TN`z~N3UDUn^q1c)7kmn! zoK5`v-DUJjskWSVq9LtilL+6fkpXAvY^gv=(P=7HRle7q1s&uRuX7P`Bz&22+)W<% z4ivmV9bt)vcZObTWx{Gy0lFvj--HEUkM6n|Qb2*l)||Da1)CHuc;<}9;ffm^<8_2s z{K&qLfkFq@Q3C>^%^nSxlgbqQH$$>3Puk*jw&j2$tNoR7cedD(M;B3^_vB!)w(6d> zfl6Ot{GoE%T;Q3BACU!@b&waxqv0#Y)>dBiegl;jh9l13tW_bEgXPrI zHKRyf!$;J0enc>-t9ER)W|HD3-NI1P2GxuOLb1E^(=rRd7Jd9y*Xs2XUZD-D|20lU?Mb=6 z7!mPKka~Z^*N7-z$jEw)E)FYE5gU;2a;)|htbCRHoKDEaB|+E%j*g1>Jw&?oFN91_o1lh+QJ zjI0<}SwaRV>zQlOEaeN)*J|slk`4tms7et-F)fzmR=o!oVm!9k6sAX5(>ZhUWmMg; z8P8>alw12NC$zS?z#KTMI*R`BIyP1c-gVd{??7k35Pl~2Ycu1yp}NDUBZRdUl_d88 zn3$oEyNoOgvwO#QogR{bO0S2oBVNd_I#BYa%Za+#T)wlmQMD*8f;_#b8bNL_M?gy8R*G3!w0#a$Odvcx1X_U=!8q)4QjtBu7)Q-sa3R3R> zo>uP(rrUjKB#snSLf8b}YCO)0$#{A~f70ud+4@n@pbi~gK_6^}(|;n|IkL}bm%yh~ zL&MG;@vgR2w_r5m!+{6RA!tqXb@>#rR5l8>Eh zaY7!7aK1z07I3DcGY#Uon7=4Co+HN1AbdP^Axd`19v;gdaS z(<17Bzvyd>qqBA>(aysn%1=TwkeQ)d*+9`i=2)+~GIMiL*|OlG5~(vk4roj^Gy7N!I(LY` z>S)z#8jcrcU#dTdl}ze99$29wL>uO(eKNT#p`Pd~@c8%gtMcs~ z{Vos1G~w>oW0H2#1VZOgp?HnBh}I}~`^{h(3BEmr^pV|+_=@!1y=rQ^&C zj`m}2K=dgTHXj;ox-MLz9M*v~#&dzG4{L!XIE-i|xJYXEyKS0jy?bjyiff%__@L&5 zKJ6Q_g`i10pv)}kb$vf^(Y;UZReBdav5SEnGxkR(%M}CKH$9Jb%+FcRejTp)GACoQ05#U#ug(> zUz2wTz*QxR)q!;s__e?pe+s(c!&Nmu#BMa@A^A;jYK{yJRwy|0$ zoD9_+Qi8Ump9V9f(vKBx-EBi%TiD)t`yR1E-*4-Wyv80qY&Z#T>AOmg8xYQa0GstQ zx=4K39Ki&uy8gU6@_;l${s=twpQQXWbdLmf3FC7hQ&Uj6h&W0sb(JEyyC((R*c;;@ zJ-5wmrih@@UNxbLw?zy0n)>=wP_{BB=gBiUpM8>DPwMVrkDi2O$r{~Z#;(}myuWJ1IP}QoA z2CQh(2XIcrfHuH)~4gdV@wDBpQf9ZgcWtWUIR2FM2LoS2sCXz`bVk*?+R>y7b=BXoi~ExY{l zy`Gl$y=dT|?`XpxX4Kyd_@530^~^kNhCDkB}qe0(_*MR5-#(h}dh7AT#yb6R=B2 za3Mq&F6zeD&+li7`9UbB;M^F$a@R4&jh66pkXf>}qzrm@ofY~t>&%Mr4O^B@)^=@@ z1|!16P;vSZDJ;|eSk)GY3WOqjL@@2>wci-%MSl1pj}5$z zD5a=7F~%oqK{5S$xsZD7pcMhC?4)`;QqGTMK6TJuiM0gn-+VPzhcWq-x96BrF!T+R zXeaxyd(B$O+0dDj?(?1pB^|FIAK6e6lYB&(cZ2#2633#enoF&hD#2Sqr>tEUOS3tP zp50nG?v#10Z%hszA8B-xQ_~(fYZmsy8&ui($$sAlT_TEy&E_zj z4@Ky8dSmu_iPq(;$@*VHG?8~u6_dX$XoOnefW(pX?l6<|cFB;@IQ^bMc0J?k2F!)T z4(kEqmn|<*eyCZqC~fx-oI|j>7Epo(O&1c@YSVG?X=gy!Xvcr2qtB6$qM!TP0loR~ z1THRG?v%|Ya8m=)eyACj&AAY7=uBkxbO+vJvCu)|-ea*OtfzoMv$qE#6gcms-H9!dI6eu0U8u$|=y=@D%`7IRzK-Pz@=0MP zNxhLc=wSDCHD{02aV|Wvs2)g4Kgu)j%^qMhrYE5UiHAW}a|%6pn~yObL(;?^)jAU5 zx73pEx)gcDzSXScsq1%?YYL(nMuBdI_Eo2mI^8FJ+ zTIDiMUP42VX^wZ(3X~x0BV)rh7+E2|TEnbG+Czs7jEy|@-`|i=j)UxWr$=@{FH(Us zN0pq`YFdmu+o2PCip-rL=zWGTnd@ko?iK|&U5sy1yF&IbO!Orvss}lg>sp8v}xnI~hDb)%;NTZ#(H!)j&2(jZlOUKFFes3DEMO(eSl%=z(HJajHOXoF&XTAnY z0U&&dO{$wDeAk3VQ*K^8@XIpC7#=RLSOu+sL$%*%OsD|g&JV%4lp+PV*y;{c7e4_| z!dRq9ZhJeTGS;vT{?sMWa83Y@_Qz4HZDu*mjPJdKM3aP$y2>{Nrs>Q~BU?Mw3ybme zgxn&XIcO13%Y*fK!b#;2ESn2?{;Dw*#$oj8G<4=m4Zg0RI!(0;iLODq$;dmUdsUk? zTa!qRSv%0im{oqIV}#Sx7py+pwy@o|ezPr{{uNcHm}amNzwd(KNV#7hjpy#YU!@*h zACB{QkL`Md`cPX`-JkE~sxFPPbclxVnp~+l4I8HYT_nx7k!FWzu;fp_K9qfEKd8du z`SxNBv5`#+$-ye~Qhj|rC;g?>K+ zErQcUa~V<8R#F;G%9rFcL(^;BMulLlh%FCA!n9|6ycKUM>C*Gq);l`v+0$)C1DvRC z#j`AilUmosy<}t(1tft6BgO2|qZsx!FRaVf${KUqhq}@P_=J{--n$i!Tpr|1C;nq^SS~u z)ygAOwOH`!Hs>?NvNuNasNN@CJwLVQ^gr|pq<~d4& zlZV>X14J+A4PhVN;#fQm+F`UtOr5H|m&dFn2I@7&`-4tOxtCri z!yc@DftTz}Ve9p_*0{WmCzMF-`W^A|IlBf)h5VecgkT8tt`m?M7YWZf6>#addUI~~ zMr19(bL!!~9qo#UZ(M(T56q!PvZp4#JT0>7dv71Y;)6=_a8XOm4;FhbHbC_0svsF! zoF~>yooC?L$#@P1xI+_r^w~IG9riw@>454rgYR;Z0DNu#nU>XV7vtG!A#acXo0xF8 z+b-7!U^X)6BuUeJy$f|cHB?8Zk4qMuPm_zDk-IeZ8Hx%-SoIcWTjdk3CjL(D?z(?^h|# zs%Rwoxl>{?GKdm1fA1LJXFJms>0Aiws7p}CM+LiujDJIav?1UZdx_7QCXnFZP#u zQ@t#m`x8c*!k915Irlk?9=s2=Ur0~vf@kR$)F#d%|LgPiS2x-vN>A5T)2=IiD>q?i z|6F1WY4uUdlw+rs`or+(j(Zdh19NiEK_tE9O^K2%Ye|C(ou%5HrO2rpsgA+C&x?N3 zG~G5+tPa(bB+y`RHAAp3^y@pCFEx2=GmWs=ZHQSH$rlS@F1NS$kH!;K@e^%vVoVCz zk-=&ED$7`F5^-Winn9ZG4420!;)h@4fEDWlm2H5-hJ@|ya)c~XLQh!Y>gG}&fe1VJ zgVzthWmps#a*#*qvI0-0WuaQhm(^Q7saT zkLK{nUvt~A zSSytWy#3Y|MI4VZ_7dMac;LMgBKx&q@w>Yd@TN+4VPb=-$+ACNu}`ekdV4C=_E9ZI zNVZ!$A_Ya}2SP!v{dK2!(y?JN@g^x~)^g-f2?BYMwBu&q1unhg>r(96iLCCMjl<<< z@TnN{NZUcgU))5O4bpmM-{TqS^*U{x(YvqZzSTXvALbR}PqixR&l5kO==9G&yUXs- zp(!eAgbg1~Lt?Cy&!2x%x1Xx9yjy34*l?z*BDmRWTN$<5ntBUEn`moTg>?Pih+iv` zwpMwcfnh95|J{6s9|c0hb|<#JDsthFKJ>D>ynO$o#<1jcgMy{na6))Wq=BNv(}%{+ zncYF}QHmM;<2c>oy7bgxKDtTi1TW za$8bIFm!Z#i=X3fmJ2~E&``v2$P5N&G@92aK&Ks40dKUM?tCZ;zn1gG$C+ul*#p6R z%f~yC>R6u591QCpkUi-4u{v|ffyGr}5LQ}-uA%bY3Biym%~QksE?3E^7(~rl^%r-f zAg{TD(l0+zObA1dXvO`>vIB+K;)qTxC>4`?tlEEODhTs2rMC36d9SezNL+wMDWUq# z>O_Sjbb9=Sx9MPZtO`)|nS~?eZ;ZQVD9mw}#$7UOoQ;k5j-JKBLJpeJs3}xAXGh## z@;F?Mdpq${8XqR{%~f=^omW5&Ff7f@L18`~7o}muSjuXhlJkfGup~e~Hd$K!bLR34 z8oV)}`9S10s1AE~Jjc366gJP;O*o$h7YQuCN@`~4^{Fn3{ct&vMOXJk^c+Hc(etS0 zH50Bv^Mk!}EY8d8lbn~%mC(!CJwp=g?l>sv^kMrl_{OMePOcJ-Jvo8&UGMi5>27+`o)p|hojrzM zE9x}CT^D2hWg#xv=_cuwjOzcZghhKOP>+olfb2Kq=3Hb*HFJ^K` zCQDFHyV%Y4xSZcR<)#SlY#8FFMg-s)GoqNrHqGC~r>&nK)8<}vnSLjMH@Xr(ZFsO| zYJ5janoZ*nt!+ET=(w=X2j1KF<81bRY41>RZENzPk93FwQd^BmG@j984-xh5jZM>i zaVg5uo_d$1qyMaC(~*0?iD>~0gf+t&yxu>lbY9ikZrF0WJ$i_r!p(;6<7{$)Hg7EO zIEf-ifk#GxMx4BVI~@fG@zaKs`9f{v6T-r{Qa1$BIFG^~uglV*lrT(+YYQ9us#BRe zY>=KKF3cj=^TEvM3>gI4hCN}y1(I4IY;uy8U`DlXEa_p!>es0@#x7Z3XB`?87y3xO++{}}Yx`80btdvES?wDg8LvS*Kqa|B;j0;`e#f2a;gd>5ARsF%Bun}L=?V+ zp)#AA*Ki$O8tQNU&d=2^5N3@ zFUZUVh?_VL6B^^B9St50g{`9jUq1rzfKgdTq_YF@I#g^7+Yk+Vg^tw{1C{s{Tni9c zGPf+XLl2Sj`dr^qMdE!4oi4&6FRkALCBAi^EE(hQiK5j;XbE+4q(ocbvR%H%E3uIgs0TeNnn!>&G)c)BL#eHB&!Tj>zvDnXsy!x}CF#{7DY- z@_%6efJ>ILffXIR-(k)Y;qS?eR_jEtdO&E-eE;0UE{g3`gJ~8 zu12mo;qy7(TWQx^<@E~ew*^gmrATo&eDY^o;?Q|-YNT}y8k1W%Q=H?uU>fP&E=wLK zCDSrTr?F(m5tO|DrhD-gA)M4u$uo59XAn;*Woh`>+2A<7o12uji6L$@i_HmUyn{JN z|BzZorIVrSA7Et6F}GyCa+=9(n~dvaAXbN!$oB4{*J5S#9oDI7HIQ*-tkxBWz?6r- z6x`F?)ULBfO3wdZ8S=wyok zDCyKV+3oTOK^Tawzd-D`(jT&ixi(*0i-mE#g&fs^FNw2kos zWN>8exC?RtQv?Kt`HPpmECkCr186LjyN5Yq2H0VDzBa-P?aY$WaTf0_pF|7K;_teHedj;uFL^U+p;A>Q z%dU`-6qhzDAun$;b9?I~S4~@+EWh^emzngL3>WgwxFhohy}94^?%=W$8L~6b-NLcM>4Dg(V9fv@SgGrwZZu54|UPz$1O!W*qPO0Z4xH9SIHpBhBk!5_GPox*j9lwhlLYt%D{tUxug_VRz=i$s@r=M=&s=z+f}Ea%*ok4L4=6$tdy zy4z+HJhc`Yo6?`-okG4fqcmVqc&0vv(A#H=)ay3y#crE?>!wBucqlNHw6WXgG0vHn z%8HQLOWO1pb=xhXA9CCiVrc2&zxFp^d`M-JMG$ff{ul*mhC31}Y}D+b#*g1?PS6Ww zF4_c=S7xD1J~JBiyr{T^()@|B?g#|ymEqZxo4bks;jsP{L(z@V5%$M6`?t7?zbT{m z(`nQ14wZi|lljwu@9zhS|6cZMk@&5m-%hW7UH*F+{qF#OvhROa_S-xB*E;^atmco( z|MhH7uZ*v-OS&9BWfr1xzmnjk#Nu=o8|#XA zIoqyxXyUg#K|Kwu1;<6g4z(rIm<6yoruJXYeDy=)7ICy3Hp;@WWWeWaEvGD@BPY|K z12`7V7-Y|FD>aA)U9BVho7UGdp?O=-!WBY=pQt0g$~Ixq;dcX`yenETxzhDl^7={G z566p3ka161Onn81)|6+XjOM%R><&g$RA0~W>hnH%!}}}ynujoK7jX49yfX`Au>h6X zsx!}i)EV5or_`@?a#3_PPx*1^V)e$n+>#g8JO!Zx48+2anD#n?$kxj&M*Fz$40uPG zRO02q-Z11)P|S6nBRtqI1MAk%l8KoVq+rEwdHKP<)Pp7$k9|Ta@;{LHUF8;-67|aZvmrH-CfjXAX>i#`(p9@rN)%{5#vnUpX=U8R@SUng0#a zpE)xA8RxGS<@^oKUpX`W8R@SUE&L7ApE)%C8Rr*^#vg+5=HJ`s-#9gX1N)s8;?IN; zzfv9k5QR5?<=FWBCH`IW&x8cO2qpfIXh;CycY2BcsrBa^(_ay`e~3BspQ#J}r{15F zHGXBF{vkt%f5}JvpNfBu2mb2=He>xYv*UNoe~K6WL%!nwbM)}^YR~J c`|%%UX+;@mm|wS{1JD5`PyoP&@GqtR2cFszVgLXD From 7c50e771e20b670484e1b5c734ef263f17b1f119 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Fri, 7 Jul 2023 23:45:36 -0500 Subject: [PATCH 07/10] refactor/clean up, move to separate templates per profile --- playbook.yml | 11 ++-- roles/tuned_amdgpu/defaults/main.yml | 1 + roles/tuned_amdgpu/files/profile-common.sh | 35 +++++++++++ roles/tuned_amdgpu/tasks/main.yml | 31 ++++++++-- .../templates/amdgpu-profile-default.sh.j2 | 36 ++++++++++++ .../templates/amdgpu-profile-overclock.sh.j2 | 58 +++++++++++++++++++ ...-clock.sh.j2 => amdgpu-profile-peak.sh.j2} | 50 +--------------- 7 files changed, 164 insertions(+), 58 deletions(-) create mode 100644 roles/tuned_amdgpu/files/profile-common.sh create mode 100644 roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 create mode 100644 roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 rename roles/tuned_amdgpu/templates/{amdgpu-clock.sh.j2 => amdgpu-profile-peak.sh.j2} (61%) diff --git a/playbook.yml b/playbook.yml index ec0ffab..eec99dd 100644 --- a/playbook.yml +++ b/playbook.yml @@ -11,16 +11,17 @@ # the connected AMD GPU is automatically discovered - assumes one # on swap to other AMD cards to avoid instability: # 'rm -rfv /etc/tuned/*amdgpu*' - gpu_clock_min: "2200" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile - gpu_clock_max: "2725" # default somewhere around 2529 to 2660 + gpu_clock_min: "750" # default 500, for best performance: near maximum. applies with 'overclock' tuned profile + gpu_clock_max: "2675" # default somewhere around 2529 to 2660 gpumem_clock_static: "1075" gpu_power_multi: default: 0.869969040247678 # 281W - real default -# overclock: 0.928792569659443 # 300W - slight boost - overclock: 1.0 # 323W - full board capability + overclock: 0.928792569659443 # 300W - slight boost +# overclock: 1.0 # 323W - full board capability # optional, applies offset (+/-) to GPU voltage by provided mV # gpu_mv_offset: "-25" - gu_mv_offset: "+75" # add 50mV or 0.075V + # gpu_mv_offset: "+50" # add 50mV or 0.05V + gpu_mv_offset: "+25" # add 25mV or 0.025V # '-50' undervolts GPU core voltage 50mV or 0.05V # mostly untested, there be dragons/instability # diff --git a/roles/tuned_amdgpu/defaults/main.yml b/roles/tuned_amdgpu/defaults/main.yml index 14a9581..6792dcb 100644 --- a/roles/tuned_amdgpu/defaults/main.yml +++ b/roles/tuned_amdgpu/defaults/main.yml @@ -9,3 +9,4 @@ profile_name: "{{ item.0 }}" amdgpu_profiles: - default - overclock + - peak diff --git a/roles/tuned_amdgpu/files/profile-common.sh b/roles/tuned_amdgpu/files/profile-common.sh new file mode 100644 index 0000000..5970513 --- /dev/null +++ b/roles/tuned_amdgpu/files/profile-common.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# +# 'common' file sourced by other scripts under tuned profile +# +# dynamically determine the connected GPU using the DRM subsystem +CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) + +function get_hwmon_dir() { + CARD_DIR="/sys/class/drm/${1}/device/" + for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do + if [[ -f "${CANDIDATE}"/power1_cap ]]; then + # found a valid hwmon dir + echo "${CANDIDATE}" + fi + done +} + +# determine the hwmon directory +HWMON_DIR=$(get_hwmon_dir "${CARD}") + +# read all of the power profiles, used to get the IDs for assignment later +PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) + +# get power capability; later used determine limits +read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max + +# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl +# improves allocation latency +echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled + +# export determinations +export CARD +export HWMON_DIR +export PROFILE_MODES +export POWER_CAP diff --git a/roles/tuned_amdgpu/tasks/main.yml b/roles/tuned_amdgpu/tasks/main.yml index a7bc037..4cc4e42 100644 --- a/roles/tuned_amdgpu/tasks/main.yml +++ b/roles/tuned_amdgpu/tasks/main.yml @@ -28,6 +28,14 @@ when: (fed_ppdtuned_swap is not defined) or ('tuned' not in ansible_facts.packages) become: true +- name: Ensure dynamic tuning is disabled + ansible.builtin.lineinfile: + path: /etc/tuned/tuned-main.conf + regexp: '^dynamic_tuning.*=' + line: 'dynamic_tuning = 0' + notify: Restart tuned + become: true + - name: Create custom profile directories ansible.builtin.file: state: directory @@ -38,20 +46,31 @@ - "{{ base_profiles }}" become: true -- name: Template AMDGPU control/reset scripts +- name: Copy 'common' AMDGPU script for all profiles + ansible.builtin.copy: + src: profile-common.sh + dest: "/etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-common.sh" + mode: "0644" # sourced, doesn't require executable bit + owner: root + group: root + notify: Restart tuned + with_nested: + - "{{ amdgpu_profiles }}" + - "{{ base_profiles }}" + become: true + +- name: Template custom AMDGPU profile scripts ansible.builtin.template: - src: templates/amdgpu-clock.sh.j2 + src: amdgpu-profile-{{ item.0 }}.sh.j2 dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/amdgpu-clock.sh owner: root group: root mode: "0755" - with_nested: - - "{{ amdgpu_profiles }}" - - "{{ base_profiles }}" + loop: "{{ amdgpu_profiles | product(base_profiles) | list }}" notify: Restart tuned become: true -- name: Template custom tuned profiles +- name: Template tuned.conf for custom profiles ansible.builtin.template: src: templates/tuned.conf.j2 dest: /etc/tuned/{{ item.1 }}-amdgpu-{{ item.0 }}/tuned.conf diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 new file mode 100644 index 0000000..4bd1282 --- /dev/null +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-default.sh.j2 @@ -0,0 +1,36 @@ +#!/bin/bash +# script for tuned AMDGPU clock control +# configures GPU power/clock characteristics +# clocks/power in 3D are dynamic based on need/usage +# +# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default +# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR) +# +# rendered by Ansible with environment-appropriate values: +# card #, eg: card0 +# path to discovered sysfs device files (power/clock/voltage control) +# +# AMDGPU driver/sysfs references: +# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html +# https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh + +{# begin the templated script for 'default' profiles to reset state #} +# set control mode back to auto +# attempts to dynamically set optimal power profile for (load) conditions +echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + +# reset any existing profile clock changes +echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# adjust power limit using multiplier against board capability +POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" + +# extract the power-saving profile ID number +PROF_DEFAULT_NUM=$(/usr/bin/awk '$0 ~ /BOOTUP_DEFAULT.*:/ {print $1}' <<< "$PROFILE_MODES") + +# reset power/clock heuristics to power-saving +echo "${PROF_DEFAULT_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 new file mode 100644 index 0000000..1f0aa3a --- /dev/null +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-overclock.sh.j2 @@ -0,0 +1,58 @@ +#!/bin/bash +# script for tuned AMDGPU clock control +# configures GPU power/clock characteristics +# clocks/power in 3D are dynamic based on need/usage +# +# for 'amdgpu-default' tuned profiles, this will reset the characteristics to default +# for others this will apply overclocking settings -- leaving clock choices to the associated power profile (eg: VR) +# +# rendered by Ansible with environment-appropriate values: +# card #, eg: card0 +# path to discovered sysfs device files (power/clock/voltage control) +# +# AMDGPU driver/sysfs references: +# https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html +# https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh + +{# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} +# set the minimum GPU clock - for best performance, this should be near the maximum +# RX6000 series power management *sucks* +echo 's 0 {{ gpu_clock_min }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# set the maximum GPU clock +echo 's 1 {{ gpu_clock_max }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# set the GPU *memory* clock +# normally this would appear disregarded, memory clocked at the minimum allowed by the overdrive (OD) range +# it follows the core clock; if both 0/1 profiles for _it_ are high enough, the memory will follow +echo 'm 1 {{ gpumem_clock_static }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage +{% if gpu_mv_offset is defined %} + +# offset GPU voltage {{ gpu_mv_offset }}mV +echo 'vo {{ gpu_mv_offset }}' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage +{% endif %} + +# commit the changes +echo 'c' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage + +# force GPU core and memory into highest clocks (fix flickering and poor power management) +# set manual control mode +# allows control via 'pp_dpm_mclk', 'pp_dpm_sclk', 'pp_dpm_pcie', 'pp_dpm_fclk', and 'pp_power_profile_mode' files +echo 'manual' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level + +# adjust power limit using multiplier against board capability +POWER_LIM_OC=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.overclock }} 'BEGIN {printf "%.0f", (m*n)}') +echo "$POWER_LIM_OC" | tee "${HWMON_DIR}/power1_cap" + +# avoid display flickering, force OC'd memory to highest clock +echo '3' | tee /sys/class/drm/"${CARD}"/device/pp_dpm_mclk + +# extract the VR power profile ID number +PROF_VR_NUM=$(/usr/bin/awk '$0 ~ /VR.*:/ {print $1}' <<< "$PROFILE_MODES") + +# force 'overclocked' profile to 'VR' power/clock heuristics +# latency/frame timing seemed favorable with relatively-close minimum clocks +echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode diff --git a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 similarity index 61% rename from roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 rename to roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 index cc2dd2a..14105a8 100644 --- a/roles/tuned_amdgpu/templates/amdgpu-clock.sh.j2 +++ b/roles/tuned_amdgpu/templates/amdgpu-profile-peak.sh.j2 @@ -13,53 +13,10 @@ # AMDGPU driver/sysfs references: # https://01.org/linuxgraphics/gfx-docs/drm/gpu/amdgpu.html # https://docs.kernel.org/gpu/amdgpu/thermal.html +# +# start by including the 'common' script; determines card/hwmon dir/power profiles/power capability +. $(dirname "${BASH_SOURCE[0]}")/amdgpu-common.sh -{# done this way to avoid issues with the card number possibly shifting after playbook run #} -# dynamically determine the connected GPU using the DRM subsystem -CARD=$(/usr/bin/grep -ls ^connected /sys/class/drm/*/status | /usr/bin/grep -o 'card[0-9]' | /usr/bin/sort | /usr/bin/uniq | /usr/bin/sort -h | /usr/bin/tail -1) - -function get_hwmon_dir() { - CARD_DIR="/sys/class/drm/${1}/device/" - for CANDIDATE in "${CARD_DIR}"/hwmon/hwmon*; do - if [[ -f "${CANDIDATE}"/power1_cap ]]; then - # found a valid hwmon dir - echo "${CANDIDATE}" - fi - done -} - -# determine the hwmon directory -HWMON_DIR=$(get_hwmon_dir "${CARD}") - -# read all of the power profiles, used to get the IDs for assignment later -PROFILE_MODES=$(< /sys/class/drm/"${CARD}"/device/pp_power_profile_mode) - -# get power capability; later used determine limits -read -r -d '' POWER_CAP < "$HWMON_DIR"/power1_cap_max - -# enable THP; profile enables the 'vm.compaction_proactiveness' sysctl -# improves allocation latency -echo 'always' | tee /sys/kernel/mm/transparent_hugepage/enabled - -{# begin the templated script for 'default' profiles to reset state #} -{% if 'default' in profile_name %} -# set control mode back to auto -# attempts to dynamically set optimal power profile for (load) conditions -echo 'auto' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level - -# reset any existing profile clock changes -echo 'r' | tee /sys/class/drm/"${CARD}"/device/pp_od_clk_voltage - -# adjust power limit using multiplier against board capability -POWER_LIM_DEFAULT=$(/usr/bin/awk -v m="$POWER_CAP" -v n={{ gpu_power_multi.default }} 'BEGIN {printf "%.0f", (m*n)}') -echo "$POWER_LIM_DEFAULT" | tee "${HWMON_DIR}/power1_cap" - -# extract the power-saving profile ID number -PROF_POWER_SAVING_NUM=$(/usr/bin/awk '$0 ~ /POWER_SAVING.*:/ {print $1}' <<< "$PROFILE_MODES") - -# reset power/clock heuristics to power-saving -echo "${PROF_POWER_SAVING_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mode -{% else %} {# begin the templated script for 'overclocked' AMD GPU profiles based on the existing tuned profiles #} # set the minimum GPU clock - for best performance, this should be near the maximum # RX6000 series power management *sucks* @@ -107,4 +64,3 @@ echo "${PROF_VR_NUM}" | tee /sys/class/drm/"${CARD}"/device/pp_power_profile_mod # ref: https://gitlab.freedesktop.org/drm/amd/-/issues/1500 # followup: doesn't work that well in practice, still flaky on clocks/frame times #echo 'high' | tee /sys/class/drm/"${CARD}"/device/power_dpm_force_performance_level -{% endif %} From e45aa26eaa2f81298a4f292c8eeccc689efe3cf2 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Fri, 7 Jul 2023 23:50:28 -0500 Subject: [PATCH 08/10] reflect new output "peak" profile --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95fd9a8..2453500 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ An example of the output/provided profiles follow |:---|---| | `balanced-amdgpu-default` | Includes the (assumed) existing `balanced` tuned profile.

    Only adjusts the GPU power limit (typically lower). Clocks/voltage curve remain the default. | | `desktop-amdgpu-overclock` | Includes the (assumed) existing `desktop` tuned profile.

    Adjusts the GPU power limit, clocks, _and_ the voltage curve. | +| `desktop-amdgpu-peak` | Includes the (assumed) existing `desktop` tuned profile.

    Same as the `overclock` profile, but locks clocks to their highest configured values | ## Notable variables From 7e709007fdc83ed397b4b84faafc4a09dbd1ef24 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Fri, 7 Jul 2023 23:54:20 -0500 Subject: [PATCH 09/10] update output profile note --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2453500..f106607 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,14 @@ An attempt is made to discover the active GPU using the 'connected' state in the ~ $ grep -ls ^connected /sys/class/drm/*/status | grep -o card[0-9] | sort | uniq | sort -h | tail -1 card1 ``` - _Warning_: This is only tested with `RX6000` series GPUs, it is probable that other generations will *not* work properly. Use at your own risk! ## Profiles -An example of the output/provided profiles follow +Two _'profiles'_ are in each name: + +- before `amdgpu` is the source profile provided with `tuned` +- after `amdgpu` tells the GPU clock profile offered, outlined below | Output profile | Description | |:---|---| From 81943ed387723a0095593674d98bc197482f9984 Mon Sep 17 00:00:00 2001 From: Josh Lay Date: Sat, 8 Jul 2023 00:02:36 -0500 Subject: [PATCH 10/10] clean up table, remove duplication of maybe-current values --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index f106607..c5242b2 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,11 @@ Two _'profiles'_ are in each name: These are the variables you're likely to want to change. They are defined in [playbook.yml](playbook.yml) -| Variable | Description | In-playbook | -|------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| gpu_clock_min | Sets the minimum (dynamic) GPU clock (in `Mhz`) for the non-default `amdgpu` profiles | `700` | -| gpu_clock_max | Sets the maximum (dynamic) GPU clock (in `MHz`) for the non-default `amdgpu` profiles | `2600`, results in `2.6GHz` (rounded); mild overclock | -| gpumem_clock_static | Sets the _static_ memory clock for the GPU (in `MHz`). This is *not* the _effective_ data rate. That is a multiple of this depending on the type of VRAM.

    To avoid flickering this does *not* change dynamically with load. | `1050`, results in just over `1GHz`; mild overclock

    Actual effective clock depends on this being multiplied against the data/pump rate of the `GDDR?` GPU memory | -| gpu_mv_offset | GPU core voltage offset. Takes +/- some integer in millivolts. Can be used to both over _and_ under volt. | `-50` (undervolt `50mV` or `0.05V`) | -| base_profiles | List of base tuned profiles to clone in the new AMDGPU profiles. Defaults based on `Fedora` |

    • `balanced`
    • `desktop`
    • `latency-performance`
    • `network-latency`
    • `network-throughput`
    • `powersave`
    • `virtual-host`
      • | -| gpu_power_multi | Dictionary with two keys, `default` and `overclock`. Expects two floats to set a power limit relative to the board _capability_. Example: `1.0` is full board capability, `0.5` is 50%.| +| Variable | Description | +|------------------------|---------------------------------------------------------------------------------------| +| gpu_clock_min | Sets the minimum (dynamic) GPU clock (in `Mhz`) for the non-default `amdgpu` profiles | +| gpu_clock_max | Sets the maximum (dynamic) GPU clock (in `MHz`) for the non-default `amdgpu` profiles | +| gpumem_clock_static | Sets the _static_ memory clock for the GPU (in `MHz`). This is *not* the _effective_ data rate. That is a multiple of this depending on the type of VRAM.

      To avoid flickering this does *not* change dynamically with load. | +| gpu_mv_offset | GPU core voltage offset. Takes +/- some integer in millivolts. Can be used to both over _and_ under volt. eg: `-50` _(undervolt `50mV` or `0.05V`)_ | +| base_profiles | List of base tuned profiles to clone in the new AMDGPU profiles. Defaults based on `Fedora` | +| gpu_power_multi | Dictionary with two keys, `default` and `overclock`. Expects two floats to set a power limit relative to the board _capability_. Example: `1.0` is full board capability, `0.5` is 50%. |