jlay/VFIO
1
1
Fork 0
Code Issues Pull requests Activity

Compare commits

base: jlay:e636412b5332b106656141f5df1ed5b3628c2998
Branches Tags
jlay:master
...
compare: jlay:5ba838386af26e8ab906a791c70f9838501de6a9
Branches Tags
jlay:master

10 commits
e636412b53 ... 5ba838386a

Author SHA1 Message Date
Josh Lay
5ba838386a add topoext for proper SMT/HT support 2021-12-19 18:38:50 -06:00
Josh Lay
deadf81902 add one more anti-cheat anti-VM trick 2021-11-28 00:18:09 -06:00
Josh Lay
88effbe450 add some anti-cheat anti-VM tricks 2021-11-28 00:13:43 -06:00
Josh Lay
29b144bd8d improve cleanup 2020-11-09 19:22:44 -06:00
Josh Lay
b4c70af268 deprecate old XML, bad iothreads and old topology 2020-11-09 16:31:50 -06:00
Josh Lay
48cc31a648 add updated XML, post threadripper upgrade 2020-11-09 16:26:18 -06:00
Josh Lay
82b6aded6f update comments and topology 2020-09-24 21:32:56 -05:00
Josh Lay
139b65a2cd Merge pull request 'update comments, require less sudo' (#1) from update-win10-script into master 
Reviewed-on: https://git.jlay.dev/jlay/vfio/pulls/1
 2020-09-25 01:34:33 +00:00
Josh Lay
c252f2e1d3 update comments, require less sudo 2020-09-24 20:33:09 -05:00
Josh Lay
7bd40d21ac add DSEreset.bat 2020-07-26 23:38:39 -04:00
4 changed files with 440 additions and 37 deletions
Show stats Expand all files Collapse all files

12
DSEreset.bat Normal file
View file

@ -0,0 +1,12 @@
:: disable driver signature enforcement
:: relies on https://github.com/Mattiwatti/EfiGuard
C:\Users\e\FreeSLI-dist\tools\EfiDSEFix.exe -d
:: let it soak
TIMEOUT /T 2
:: reset the devices, load the unsigned drivers
C:\Users\e\FreeSLI-dist\tools\devcon.exe disable "PCI\VEN_10DE&DEV_1E87&SUBSYS_21833842"
C:\Users\e\FreeSLI-dist\tools\devcon.exe enable "PCI\VEN_10DE&DEV_1E87&SUBSYS_21833842"
:: re-enable driver signature enforcement
TIMEOUT /T 2
C:\Users\e\FreeSLI-dist\tools\EfiDSEFix.exe -e
PAUSE

0
windows10-1586050224.xml → old/windows10-1586050224.xml
View file

94
start-win10
View file

@ -1,63 +1,83 @@
#!/bin/bash
# requires `cset`
#
# script to create cpusets to isolate CPU threads for KVM/libvirt VMs
# requires 'cset'
# available on Fedora through my COPR:
# - https://copr.fedorainfracloud.org/coprs/jlay/cpuset/
# review notes regarding XML entries
# review comments below regarding CPUs and XML entries
#
# this script:
# - isolates host tasks from the VM CPUs
# - applies some generic tweaks
# - resets host isolation on VM shutdown
# Set the range of cores assigned to the windows VM
# on an i9-7920x this is the first 8 cores and the corresponding hyperthreads
VM_CORES='0-7,12-19'
# based on the assignments of pinned CPUs in libvirt XML, eg:
#
# define the CPUs (cores/threads) to isolate for the VM
VM_CORES='0-20,24-44'
# Tip: refer to 'virsh capabilities | grep siblings'
# this will show CPU topology information for the host
# allows one to pick optimal layout for the VM
#
# the pinned CPUs are definend like so in the VM libvirt XML:
# <vcpupin vcpu='0' cpuset='0'/>
# <vcpupin vcpu='1' cpuset='12'/>
# [...]
# <vcpupin vcpu='14' cpuset='7'/>
# <vcpupin vcpu='15' cpuset='19'/>
#
# `lscpu -e` and `virsh capabilities` are helpful for determining valid pinning settings
# on example system (i9 7920x), vCPUs 0-15 are pinned (alternating between) cores 0-7 and hyperthreads 12-19
# <vcpupin vcpu='1' cpuset='24'/>
# <vcpupin vcpu='2' cpuset='1'/>
# <vcpupin vcpu='3' cpuset='25'/>
# [...]
# <emulatorpin cpuset='10-11'/>
#
# NOTE: the pinned emulator CPUs must be included in VM_CORES
# probably any iothreads too
#
# example above is based on a threadripper 3960x:
# - one NUMA node
# - SMT enabled
#
# reduce kernel jitter
sudo sysctl vm.stat_interval=120
sudo sysctl kernel.watchdog=0
sudo sh -c 'sysctl vm.stat_interval=120 ; sysctl kernel.watchdog=0'
# isolate VM CPUs from host tasks
# VM must have the partition cset uses w/ userset below defined in libvirt XML, eg:
#
# </cputune> ... reference line
# attempt to reset shielding, we want to create it with current params
# and remove any existing slices from previous VM boots
# we want it to be created with current params, such as possibly-changed pinned CPUs
sudo cset shield --reset --sysset=host.slice --userset=windows10.slice || echo "something happened resetting shielding"
# destroy old cpusets
sudo cset set -d windows10.slice || echo "ignoring failure to destroy VM cpuset"
sudo cset set -d host.slice || echo "ignoring failure to destroy host cpuset"
# shield cores
# userset slice must match the partition defined in libvirt xml, eg:
#
# </cputune>
# <resource>
# <partition>/windows10</partition>
# </resource>
# <os> ... reference line
# first, attempt to reset shielding. we want to recreate conflicting names with set params
sudo cset shield --reset --sysset=host.slice --userset=windows10.slice
sudo cset set -d windows10.slice
# shield cores
# <os>
sudo cset shield -c $VM_CORES --sysset=host.slice --userset=windows10.slice
# start the VM
sudo virsh start windows10
virsh start windows10
# set higher priority
QEMU_PID=$(sudo ps fauxww | awk '$0 ~ /qemu.*windows1[0]/ {print $2}')
sudo renice -20 -p ${QEMU_PID}
sudo chrt -f -p 99 ${QEMU_PID}
# determine QEMU vCPU PIDs
QEMU_PIDS=$(sudo egrep 'vcpu.*pid' /var/run/libvirt/qemu/windows10.xml | cut -d\' -f4)
# set higher priority for vCPU PIDs
sudo renice -20 -p ${QEMU_PIDS}
# either of these attempts at real time seem to freeze on boot
# try a different kernel (not 5.8.11)
#for CPUPID in ${QEMU_PIDS}; do
## sudo chrt -f -p 99 ${CPUPID}
# sudo chrt -r -p 99 ${CPUPID}
#done
echo "Waiting for windows10 VM to stop before resetting cpusets"
while true; do
# chill a bit
sleep 10
# get vm state, check if it's off
VM_STATE=$(sudo virsh dominfo windows10 | awk '$1 ~ /State/ {print $NF}')
if [ "$VM_STATE" == 'off' ]; then
if test -f "/var/run/libvirt/qemu/windows10.xml"; then
VM_STATE="on" # doesn't do much (yet?), just here to fill a spot
else
VM_STATE="off"
echo -e "windows10 VM shut down, setting cpusets back to normal\n"
sudo cset shield --reset --sysset=host.slice --userset=windows10.slice
exit 0

371
windows10-threadripper-pinning.xml Normal file
View file

@ -0,0 +1,371 @@
<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
<name>windows10</name>
<uuid>ebe29b71-9205-4032-9880-9a02bdf2cc2e</uuid>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
<libosinfo:os id="http://microsoft.com/win/10"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>33554432</memory>
<currentMemory unit='KiB'>33554432</currentMemory>
<memoryBacking>
<hugepages/>
<nosharepages/>
<locked/>
</memoryBacking>
<vcpu placement='static'>32</vcpu>
<iothreads>3</iothreads>
<cputune>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='24'/>
<vcpupin vcpu='2' cpuset='1'/>
<vcpupin vcpu='3' cpuset='25'/>
<vcpupin vcpu='4' cpuset='2'/>
<vcpupin vcpu='5' cpuset='26'/>
<vcpupin vcpu='6' cpuset='3'/>
<vcpupin vcpu='7' cpuset='27'/>
<vcpupin vcpu='8' cpuset='4'/>
<vcpupin vcpu='9' cpuset='28'/>
<vcpupin vcpu='10' cpuset='5'/>
<vcpupin vcpu='11' cpuset='29'/>
<vcpupin vcpu='12' cpuset='6'/>
<vcpupin vcpu='13' cpuset='30'/>
<vcpupin vcpu='14' cpuset='7'/>
<vcpupin vcpu='15' cpuset='31'/>
<vcpupin vcpu='16' cpuset='8'/>
<vcpupin vcpu='17' cpuset='32'/>
<vcpupin vcpu='18' cpuset='9'/>
<vcpupin vcpu='19' cpuset='33'/>
<vcpupin vcpu='20' cpuset='10'/>
<vcpupin vcpu='21' cpuset='34'/>
<vcpupin vcpu='22' cpuset='11'/>
<vcpupin vcpu='23' cpuset='35'/>
<vcpupin vcpu='24' cpuset='12'/>
<vcpupin vcpu='25' cpuset='36'/>
<vcpupin vcpu='26' cpuset='13'/>
<vcpupin vcpu='27' cpuset='37'/>
<vcpupin vcpu='28' cpuset='14'/>
<vcpupin vcpu='29' cpuset='38'/>
<vcpupin vcpu='30' cpuset='15'/>
<vcpupin vcpu='31' cpuset='39'/>
<iothreadpin iothread='1' cpuset='16,40'/>
<iothreadpin iothread='2' cpuset='17-20'/>
<iothreadpin iothread='3' cpuset='41-44'/>
</cputune>
<resource>
<partition>/windows10</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-4.2'>hvm</type>
<loader readonly='yes' type='pflash'>/usr/share/edk2/ovmf/OVMF_CODE.fd</loader>
<nvram>/var/lib/libvirt/qemu/nvram/windows10_VARS.fd</nvram>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vendor_id state='on' value='ahPauJae5thu'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<reset state='on'/>
<frequencies state='on'/>
</hyperv>
<kvm>
<hidden state='on'/>
</kvm>
<pmu state='off'/>
<vmport state='off'/>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-model' check='none'>
<topology sockets='1' dies='1' cores='16' threads='2'/>
<feature policy='disable' name='hypervisor'/>
<feature policy='require' name='x2apic'/>
<feature policy='require' name='topoext'/>
</cpu>
<clock offset='localtime'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
<timer name='hypervclock' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/bin/qemu-system-x86_64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='raw' cache='writeback' io='threads' iothread='1'/>
<source file='/var/lib/libvirt/images/EFIGuard.img'/>
<target dev='vda' bus='virtio'/>
<boot order='1'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='writeback' io='threads' iothread='2'/>
<source dev='/dev/osraid/vm_windows10'/>
<target dev='vdc' bus='virtio'/>
<boot order='2'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='writeback' io='threads' iothread='3'/>
<source dev='/dev/nvme-raid0/vm_windows10-games'/>
<target dev='vde' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x11' slot='0x00' function='0x0'/>
</disk>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0x17'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x1b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x1c'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x1d'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x5'/>
</controller>
<controller type='pci' index='15' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='15' port='0x1e'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x6'/>
</controller>
<controller type='pci' index='16' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='16' port='0x1f'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x7'/>
</controller>
<controller type='pci' index='17' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='17' port='0x20'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='18' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='18' port='0x21'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x1'/>
</controller>
<controller type='pci' index='19' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='19' port='0x22'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x2'/>
</controller>
<controller type='pci' index='20' model='pcie-to-pci-bridge'>
<model name='pcie-pci-bridge'/>
<address type='pci' domain='0x0000' bus='0x12' slot='0x00' function='0x0'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<interface type='network'>
<mac address='52:54:00:20:cc:22'/>
<source network='virtbridge'/>
<model type='virtio'/>
<driver name='vhost' queues='32'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
</serial>
<console type='pty'>
<target type='serial' port='0'/>
</console>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='1'/>
</input>
<graphics type='spice' autoport='yes'>
<listen type='address'/>
<image compression='off'/>
</graphics>
<sound model='ich9'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
</sound>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
</video>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x21' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x21' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x21' slot='0x00' function='0x2'/>
</source>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x21' slot='0x00' function='0x3'/>
</source>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x4b' slot='0x00' function='0x0'/>
</source>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x4b' slot='0x00' function='0x1'/>
</source>
<address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x4b' slot='0x00' function='0x2'/>
</source>
<address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x4b' slot='0x00' function='0x3'/>
</source>
<address type='pci' domain='0x0000' bus='0x0e' slot='0x00' function='0x0'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x1038'/>
<product id='0x1280'/>
</source>
<address type='usb' bus='0' port='4'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x1038'/>
<product id='0x1283'/>
</source>
<address type='usb' bus='0' port='5'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x046d'/>
<product id='0xc53a'/>
</source>
<address type='usb' bus='0' port='6'/>
</hostdev>
<hostdev mode='subsystem' type='usb' managed='yes'>
<source>
<vendor id='0x1e7d'/>
<product id='0x3098'/>
</source>
<address type='usb' bus='0' port='7'/>
</hostdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
</memballoon>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<address type='pci' domain='0x0000' bus='0x0f' slot='0x00' function='0x0'/>
</rng>
</devices>
<qemu:commandline>
<qemu:arg value='-overcommit'/>
<qemu:arg value='cpu-pm=on'/>
<qemu:arg value='-set'/>
<qemu:arg value='device.virtio-disk0.discard=on'/>
<qemu:arg value='-set'/>
<qemu:arg value='device.virtio-disk2.discard=on'/>
<qemu:arg value='-set'/>
<qemu:arg value='device.virtio-disk4.discard=on'/>
</qemu:commandline>
</domain>