diff --git a/README.md b/README.md index 5b438e6..3210ae7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,17 @@ # legion-7-vfio -VFIO scripts, notes, and configs for the Lenovo Legion 7 \ No newline at end of file +VFIO scripts, notes, and configs for the Lenovo Legion 7 (2021). + +The `notes` directory mainly contains command outputs and other useful pieces of information such as IOMMU groups on various kernels. + +The `scripts` directory contains: + +- `iommu.sh` which can be used to see your system's IOMMU groups +- multiple QEMU hook scripts + +The `xml` directory contains the final XML files for two Windows 10 virtual machines. + +- `Win10-nopass.xml` is a virtual machine without any passthrough +- `Win10.xml` is a virtual machine with GPU, Audio, Network Card, Keyboard, Touchpad, and USB Mouse passthrough + +Full guide on how to create a Windows 10 gaming virtual machine available on my [blog](https://blog.karaolidis.com/vfio/). diff --git a/notes/iommu-groups-linux-vfio.txt b/notes/iommu-groups-linux-vfio.txt new file mode 100644 index 0000000..6f16aca --- /dev/null +++ b/notes/iommu-groups-linux-vfio.txt @@ -0,0 +1,31 @@ +IOMMU Group 0 00:01.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 10 00:18.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166a] +IOMMU Group 10 00:18.1 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166b] +IOMMU Group 10 00:18.2 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166c] +IOMMU Group 10 00:18.3 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166d] +IOMMU Group 10 00:18.4 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166e] +IOMMU Group 10 00:18.5 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166f] +IOMMU Group 10 00:18.6 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:1670] +IOMMU Group 10 00:18.7 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:1671] +IOMMU Group 11 01:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA104M [GeForce RTX 3070 Mobile / Max-Q] [10de:24dd] (rev a1) +IOMMU Group 12 01:00.1 Audio device [0403]: NVIDIA Corporation GA104 High Definition Audio Controller [10de:228b] (rev a1) +IOMMU Group 13 02:00.0 Non-Volatile memory controller [0108]: Samsung Electronics Co Ltd NVMe SSD Controller PM9A1/PM9A3/980PRO [144d:a80a] +IOMMU Group 14 03:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 15) +IOMMU Group 15 04:00.0 Network controller [0280]: Intel Corporation Wi-Fi 6 AX200 [8086:2723] (rev 1a) +IOMMU Group 16 05:00.0 Non-Essential Instrumentation [1300]: Advanced Micro Devices, Inc. [AMD] Zeppelin/Raven/Raven2 PCIe Dummy Function [1022:145a] (rev c5) +IOMMU Group 17 05:00.2 Encryption controller [1080]: Advanced Micro Devices, Inc. [AMD] Family 17h (Models 10h-1fh) Platform Security Processor [1022:15df] +IOMMU Group 18 05:00.3 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Renoir USB 3.1 [1022:1639] +IOMMU Group 19 05:00.4 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Renoir USB 3.1 [1022:1639] +IOMMU Group 1 00:01.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1633] +IOMMU Group 20 05:00.6 Audio device [0403]: Advanced Micro Devices, Inc. [AMD] Family 17h (Models 10h-1fh) HD Audio Controller [1022:15e3] +IOMMU Group 21 06:00.0 SATA controller [0106]: Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode] [1022:7901] (rev 81) +IOMMU Group 22 06:00.1 SATA controller [0106]: Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode] [1022:7901] (rev 81) +IOMMU Group 2 00:01.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 3 00:02.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 4 00:02.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 5 00:02.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 6 00:08.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 7 00:08.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir Internal PCIe GPP Bridge to Bus [1022:1635] +IOMMU Group 8 00:08.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir Internal PCIe GPP Bridge to Bus [1022:1635] +IOMMU Group 9 00:14.0 SMBus [0c05]: Advanced Micro Devices, Inc. [AMD] FCH SMBus Controller [1022:790b] (rev 51) +IOMMU Group 9 00:14.3 ISA bridge [0601]: Advanced Micro Devices, Inc. [AMD] FCH LPC Bridge [1022:790e] (rev 51) diff --git a/notes/iommu-groups-linux.txt b/notes/iommu-groups-linux.txt new file mode 100644 index 0000000..a5f3a5f --- /dev/null +++ b/notes/iommu-groups-linux.txt @@ -0,0 +1,31 @@ +IOMMU Group 0 00:01.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 10 02:00.0 Non-Volatile memory controller [0108]: Samsung Electronics Co Ltd NVMe SSD Controller PM9A1/PM9A3/980PRO [144d:a80a] +IOMMU Group 11 03:00.0 Ethernet controller [0200]: Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express Gigabit Ethernet Controller [10ec:8168] (rev 15) +IOMMU Group 12 04:00.0 Network controller [0280]: Intel Corporation Wi-Fi 6 AX200 [8086:2723] (rev 1a) +IOMMU Group 1 00:01.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1633] +IOMMU Group 2 00:01.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 3 00:02.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 4 00:02.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 5 00:02.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe GPP Bridge [1022:1634] +IOMMU Group 6 00:08.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Renoir PCIe Dummy Host Bridge [1022:1632] +IOMMU Group 6 00:08.1 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir Internal PCIe GPP Bridge to Bus [1022:1635] +IOMMU Group 6 00:08.2 PCI bridge [0604]: Advanced Micro Devices, Inc. [AMD] Renoir Internal PCIe GPP Bridge to Bus [1022:1635] +IOMMU Group 6 05:00.0 Non-Essential Instrumentation [1300]: Advanced Micro Devices, Inc. [AMD] Zeppelin/Raven/Raven2 PCIe Dummy Function [1022:145a] (rev c5) +IOMMU Group 6 05:00.2 Encryption controller [1080]: Advanced Micro Devices, Inc. [AMD] Family 17h (Models 10h-1fh) Platform Security Processor [1022:15df] +IOMMU Group 6 05:00.3 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Renoir USB 3.1 [1022:1639] +IOMMU Group 6 05:00.4 USB controller [0c03]: Advanced Micro Devices, Inc. [AMD] Renoir USB 3.1 [1022:1639] +IOMMU Group 6 05:00.6 Audio device [0403]: Advanced Micro Devices, Inc. [AMD] Family 17h (Models 10h-1fh) HD Audio Controller [1022:15e3] +IOMMU Group 6 06:00.0 SATA controller [0106]: Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode] [1022:7901] (rev 81) +IOMMU Group 6 06:00.1 SATA controller [0106]: Advanced Micro Devices, Inc. [AMD] FCH SATA Controller [AHCI mode] [1022:7901] (rev 81) +IOMMU Group 7 00:14.0 SMBus [0c05]: Advanced Micro Devices, Inc. [AMD] FCH SMBus Controller [1022:790b] (rev 51) +IOMMU Group 7 00:14.3 ISA bridge [0601]: Advanced Micro Devices, Inc. [AMD] FCH LPC Bridge [1022:790e] (rev 51) +IOMMU Group 8 00:18.0 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166a] +IOMMU Group 8 00:18.1 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166b] +IOMMU Group 8 00:18.2 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166c] +IOMMU Group 8 00:18.3 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166d] +IOMMU Group 8 00:18.4 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166e] +IOMMU Group 8 00:18.5 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:166f] +IOMMU Group 8 00:18.6 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:1670] +IOMMU Group 8 00:18.7 Host bridge [0600]: Advanced Micro Devices, Inc. [AMD] Device [1022:1671] +IOMMU Group 9 01:00.0 VGA compatible controller [0300]: NVIDIA Corporation GA104M [GeForce RTX 3070 Mobile / Max-Q] [10de:24dd] (rev a1) +IOMMU Group 9 01:00.1 Audio device [0403]: NVIDIA Corporation GA104 High Definition Audio Controller [10de:228b] (rev a1) diff --git a/scripts/iommu.sh b/scripts/iommu.sh new file mode 100644 index 0000000..6262cdd --- /dev/null +++ b/scripts/iommu.sh @@ -0,0 +1,6 @@ +#!/bin/bash +for d in /sys/kernel/iommu_groups/*/devices/*; do + n=${d#*/iommu_groups/*}; n=${n%%/*} + printf 'IOMMU Group %s ' "$n" + lspci -nns "${d##*/}" +done diff --git a/scripts/prepare/begin/alloc_hugepages.sh b/scripts/prepare/begin/alloc_hugepages.sh new file mode 100644 index 0000000..ca9af58 --- /dev/null +++ b/scripts/prepare/begin/alloc_hugepages.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +## Load the config file +source "/etc/libvirt/hooks/kvm.conf" + +## Calculate number of hugepages to allocate from memory (in MB) +HUGEPAGES="$(($MEMORY/$(($(grep Hugepagesize /proc/meminfo | awk '{print $2}')/1024))))" + +echo "Allocating hugepages..." +echo $HUGEPAGES > /proc/sys/vm/nr_hugepages +ALLOC_PAGES=$(cat /proc/sys/vm/nr_hugepages) + +TRIES=0 +while (( $ALLOC_PAGES != $HUGEPAGES && $TRIES < 1000 )) +do + echo 1 > /proc/sys/vm/compact_memory ## defrag ram + echo $HUGEPAGES > /proc/sys/vm/nr_hugepages + ALLOC_PAGES=$(cat /proc/sys/vm/nr_hugepages) + echo "Succesfully allocated $ALLOC_PAGES / $HUGEPAGES" + let TRIES+=1 +done + +if [ "$ALLOC_PAGES" -ne "$HUGEPAGES" ] +then + echo "Not able to allocate all hugepages. Reverting..." + echo 0 > /proc/sys/vm/nr_hugepages + exit 1 +fi diff --git a/scripts/prepare/begin/cpu_mode_performance.sh b/scripts/prepare/begin/cpu_mode_performance.sh new file mode 100644 index 0000000..7a1c821 --- /dev/null +++ b/scripts/prepare/begin/cpu_mode_performance.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +## Enable CPU governor performance mode +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +for file in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "performance" > $file; done +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor diff --git a/scripts/prepare/begin/start.sh b/scripts/prepare/begin/start.sh new file mode 100644 index 0000000..0917147 --- /dev/null +++ b/scripts/prepare/begin/start.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -x + +# Load Variables +source "/etc/libvirt/hooks/kvm.conf" + +# Stop LightDM +systemctl stop lightdm.service +sleep 2 + +# Unbind VTconsoles +echo 0 > /sys/class/vtconsole/vtcon0/bind +echo 0 > /sys/class/vtconsole/vtcon1/bind + +# Unbind EFI-Framebuffer +echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind + +# Avoid a race condition by waiting a couple of seconds. This can be calibrated to be shorter or longer if required for your system +sleep 2 + +# Unload all Nvidia drivers +modprobe -r nvidia_drm +modprobe -r nvidia_modeset +modprobe -r drm_kms_helper +modprobe -r nvidia +modprobe -r drm + +# Unbind the GPU from display driver +virsh nodedev-detach $VIRSH_GPU_VIDEO +virsh nodedev-detach $VIRSH_GPU_AUDIO + +# Load VFIO kernel module +modprobe vfio +modprobe vfio_pci +modprobe vfio_iommu_type1 diff --git a/scripts/release/end/cpu_mode_schedutil.sh b/scripts/release/end/cpu_mode_schedutil.sh new file mode 100644 index 0000000..5e471f0 --- /dev/null +++ b/scripts/release/end/cpu_mode_schedutil.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +## Enable CPU governor on-demand mode +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +for file in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "schedutil" > $file; done +cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor diff --git a/scripts/release/end/dealloc_hugepages.sh b/scripts/release/end/dealloc_hugepages.sh new file mode 100644 index 0000000..2e35d1c --- /dev/null +++ b/scripts/release/end/dealloc_hugepages.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +## Load the config file +source "/etc/libvirt/hooks/kvm.conf" + +echo 0 > /proc/sys/vm/nr_hugepages diff --git a/scripts/release/end/revert.sh b/scripts/release/end/revert.sh new file mode 100644 index 0000000..3748ec0 --- /dev/null +++ b/scripts/release/end/revert.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -x + +# Load the config file with our environmental variables +source "/etc/libvirt/hooks/kvm.conf" + +# Unload VFIO-PCI Kernel Driver +modprobe -r vfio_pci +modprobe -r vfio_iommu_type1 +modprobe -r vfio + +# Re-Bind GPU to our display drivers +virsh nodedev-reattach $VIRSH_GPU_VIDEO +virsh nodedev-reattach $VIRSH_GPU_AUDIO + +# Rebind VT consoles +echo 1 > /sys/class/vtconsole/vtcon0/bind +echo 1 > /sys/class/vtconsole/vtcon1/bind + +# Read our nvidia configuration when before starting our graphics +nvidia-xconfig --query-gpu-info > /dev/null 2>&1 + +# Re-Bind EFI-Framebuffer +echo "efi-framebuffer.0" > /sys/bus/platform/drivers/efi-framebuffer/bind + +# Load nvidia drivers +modprobe nvidia_drm +modprobe nvidia_modeset +modprobe drm_kms_helper +modprobe nvidia +modprobe drm + +# Restart Display Manager +systemctl start lightdm.service diff --git a/xml/Win10-nopass.xml b/xml/Win10-nopass.xml new file mode 100644 index 0000000..d50cb67 --- /dev/null +++ b/xml/Win10-nopass.xml @@ -0,0 +1,241 @@ + + + + Win10-nopass + 550b0a1a-a823-467f-9632-a100834cb900 + Windows 10 - No Passthrough + + + + + + 25165824 + 25165824 + + + + 12 + 2 + + + + + + + + + + + + + + + + + + + hvm + /usr/share/edk2-ovmf/x64/OVMF_CODE.fd + /var/lib/libvirt/qemu/nvram/Win10-nopass_VARS.fd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + destroy + restart + destroy + + + + + + /usr/bin/qemu-system-x86_64 + + + + + +
+ + + + + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + +
+ + +
+ + + + + +
+ + + +
+ + +
+ + +
+ + + + + + + + + +
+ +