From da9c802f52203f5190a9cae4ee4c92e4eff52ae9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 7 Oct 2025 15:36:25 -0700 Subject: [PATCH 01/78] KVM: selftests: Rename "guest_paddr" variables to "gpa" ANBZ: #40036 commit 83e0e12219a402bf7b8fdef067e51f945a92fd26 upstream. Rename "guest_paddr" variables in vm_userspace_mem_region_add() and vm_mem_add() to KVM's de facto standard "gpa", both for consistency and to shorten line lengths. Opportunistically fix the indentation of the vm_userspace_mem_region_add() declaration. Intel-SIG: commit 83e0e12219a4 KVM: selftests: Rename "guest_paddr" variables to "gpa" Link: https://patch.msgid.link/20251007223625.369939-1-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- .../testing/selftests/kvm/include/kvm_util.h | 10 ++-- tools/testing/selftests/kvm/lib/kvm_util.c | 46 +++++++++---------- 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 282b2944cd0f..5a5d6adf03bb 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -550,12 +550,12 @@ int __vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flag uint32_t guest_memfd, uint64_t guest_memfd_offset); void vm_userspace_mem_region_add(struct kvm_vm *vm, - enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, uint64_t npages, - uint32_t flags); + enum vm_mem_backing_src_type src_type, + uint64_t gpa, uint32_t slot, uint64_t npages, + uint32_t flags); void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, uint64_t npages, - uint32_t flags, int guest_memfd_fd, uint64_t guest_memfd_offset); + uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags, + int guest_memfd_fd, uint64_t guest_memfd_offset); #ifndef vm_arch_has_protected_memory static inline bool vm_arch_has_protected_memory(struct kvm_vm *vm) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 7f6c2de759e6..8b812104d5fa 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -891,8 +891,8 @@ void vm_set_user_memory_region2(struct kvm_vm *vm, uint32_t slot, uint32_t flags /* FIXME: This thing needs to be ripped apart and rewritten. */ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, uint64_t npages, - uint32_t flags, int guest_memfd, uint64_t guest_memfd_offset) + uint64_t gpa, uint32_t slot, uint64_t npages, uint32_t flags, + int guest_memfd, uint64_t guest_memfd_offset) { int ret; struct userspace_mem_region *region; @@ -904,30 +904,29 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, "Number of guest pages is not compatible with the host. " "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages)); - TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical " + TEST_ASSERT((gpa % vm->page_size) == 0, "Guest physical " "address not on a page boundary.\n" - " guest_paddr: 0x%lx vm->page_size: 0x%x", - guest_paddr, vm->page_size); - TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1) + " gpa: 0x%lx vm->page_size: 0x%x", + gpa, vm->page_size); + TEST_ASSERT((((gpa >> vm->page_shift) + npages) - 1) <= vm->max_gfn, "Physical range beyond maximum " "supported physical address,\n" - " guest_paddr: 0x%lx npages: 0x%lx\n" + " gpa: 0x%lx npages: 0x%lx\n" " vm->max_gfn: 0x%lx vm->page_size: 0x%x", - guest_paddr, npages, vm->max_gfn, vm->page_size); + gpa, npages, vm->max_gfn, vm->page_size); /* * Confirm a mem region with an overlapping address doesn't * already exist. */ region = (struct userspace_mem_region *) userspace_mem_region_find( - vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1); + vm, gpa, (gpa + npages * vm->page_size) - 1); if (region != NULL) TEST_FAIL("overlapping userspace_mem_region already " "exists\n" - " requested guest_paddr: 0x%lx npages: 0x%lx " - "page_size: 0x%x\n" - " existing guest_paddr: 0x%lx size: 0x%lx", - guest_paddr, npages, vm->page_size, + " requested gpa: 0x%lx npages: 0x%lx page_size: 0x%x\n" + " existing gpa: 0x%lx size: 0x%lx", + gpa, npages, vm->page_size, (uint64_t) region->region.guest_phys_addr, (uint64_t) region->region.memory_size); @@ -941,8 +940,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, "already exists.\n" " requested slot: %u paddr: 0x%lx npages: 0x%lx\n" " existing slot: %u paddr: 0x%lx size: 0x%lx", - slot, guest_paddr, npages, - region->region.slot, + slot, gpa, npages, region->region.slot, (uint64_t) region->region.guest_phys_addr, (uint64_t) region->region.memory_size); } @@ -968,7 +966,7 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, if (src_type == VM_MEM_SRC_ANONYMOUS_THP) alignment = max(backing_src_pagesz, alignment); - TEST_ASSERT_EQ(guest_paddr, align_up(guest_paddr, backing_src_pagesz)); + TEST_ASSERT_EQ(gpa, align_up(gpa, backing_src_pagesz)); /* Add enough memory to align up if necessary */ if (alignment > 1) @@ -1028,20 +1026,18 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, region->unused_phy_pages = sparsebit_alloc(); if (vm_arch_has_protected_memory(vm)) region->protected_phy_pages = sparsebit_alloc(); - sparsebit_set_num(region->unused_phy_pages, - guest_paddr >> vm->page_shift, npages); + sparsebit_set_num(region->unused_phy_pages, gpa >> vm->page_shift, npages); region->region.slot = slot; region->region.flags = flags; - region->region.guest_phys_addr = guest_paddr; + region->region.guest_phys_addr = gpa; region->region.memory_size = npages * vm->page_size; region->region.userspace_addr = (uintptr_t) region->host_mem; ret = __vm_ioctl(vm, KVM_SET_USER_MEMORY_REGION2, ®ion->region); TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION2 IOCTL failed,\n" " rc: %i errno: %i\n" " slot: %u flags: 0x%x\n" - " guest_phys_addr: 0x%lx size: 0x%lx guest_memfd: %d\n", - ret, errno, slot, flags, - guest_paddr, (uint64_t) region->region.memory_size, + " guest_phys_addr: 0x%lx size: 0x%llx guest_memfd: %d", + ret, errno, slot, flags, gpa, region->region.memory_size, region->region.guest_memfd); /* Add to quick lookup data structures */ @@ -1063,10 +1059,10 @@ void vm_mem_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, void vm_userspace_mem_region_add(struct kvm_vm *vm, enum vm_mem_backing_src_type src_type, - uint64_t guest_paddr, uint32_t slot, - uint64_t npages, uint32_t flags) + uint64_t gpa, uint32_t slot, uint64_t npages, + uint32_t flags) { - vm_mem_add(vm, src_type, guest_paddr, slot, npages, flags, -1, 0); + vm_mem_add(vm, src_type, gpa, slot, npages, flags, -1, 0); } /* -- Gitee From b8e5b4d3df5052264dbfc104c9cbff0e13f1a5f4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 1 Dec 2025 18:03:33 -0800 Subject: [PATCH 02/78] KVM: Disallow toggling KVM_MEM_GUEST_MEMFD on an existing memslot ANBZ: #40036 commit 9935df5333aa503a18de5071f53762b65c783c4c upstream. Reject attempts to disable KVM_MEM_GUEST_MEMFD on a memslot that was initially created with a guest_memfd binding, as KVM doesn't support toggling KVM_MEM_GUEST_MEMFD on existing memslots. KVM prevents enabling KVM_MEM_GUEST_MEMFD, but doesn't prevent clearing the flag. Failure to reject the new memslot results in a use-after-free due to KVM not unbinding from the guest_memfd instance. Unbinding on a FLAGS_ONLY change is easy enough, and can/will be done as a hardening measure (in anticipation of KVM supporting dirty logging on guest_memfd at some point), but fixing the use-after-free would only address the immediate symptom. ================================================================== BUG: KASAN: slab-use-after-free in kvm_gmem_release+0x362/0x400 [kvm] Write of size 8 at addr ffff8881111ae908 by task repro/745 CPU: 7 UID: 1000 PID: 745 Comm: repro Not tainted 6.18.0-rc6-115d5de2eef3-next-kasan #3 NONE Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 Call Trace: dump_stack_lvl+0x51/0x60 print_report+0xcb/0x5c0 kasan_report+0xb4/0xe0 kvm_gmem_release+0x362/0x400 [kvm] __fput+0x2fa/0x9d0 task_work_run+0x12c/0x200 do_exit+0x6ae/0x2100 do_group_exit+0xa8/0x230 __x64_sys_exit_group+0x3a/0x50 x64_sys_call+0x737/0x740 do_syscall_64+0x5b/0x900 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x7f581f2eac31 Allocated by task 745 on cpu 6 at 9.746971s: kasan_save_stack+0x20/0x40 kasan_save_track+0x13/0x50 __kasan_kmalloc+0x77/0x90 kvm_set_memory_region.part.0+0x652/0x1110 [kvm] kvm_vm_ioctl+0x14b0/0x3290 [kvm] __x64_sys_ioctl+0x129/0x1a0 do_syscall_64+0x5b/0x900 entry_SYSCALL_64_after_hwframe+0x4b/0x53 Freed by task 745 on cpu 6 at 9.747467s: kasan_save_stack+0x20/0x40 kasan_save_track+0x13/0x50 __kasan_save_free_info+0x37/0x50 __kasan_slab_free+0x3b/0x60 kfree+0xf5/0x440 kvm_set_memslot+0x3c2/0x1160 [kvm] kvm_set_memory_region.part.0+0x86a/0x1110 [kvm] kvm_vm_ioctl+0x14b0/0x3290 [kvm] __x64_sys_ioctl+0x129/0x1a0 do_syscall_64+0x5b/0x900 entry_SYSCALL_64_after_hwframe+0x4b/0x53 Intel-SIG: commit 9935df5333aa KVM: Disallow toggling KVM_MEM_GUEST_MEMFD on an existing memslot Reported-by: Alexander Potapenko Fixes: a7800aa80ea4 ("KVM: Add KVM_CREATE_GUEST_MEMFD ioctl() for guest-specific backing memory") Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20251202020334.1171351-2-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- virt/kvm/kvm_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 88ef76ad991a..6d035f4d85a1 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2206,7 +2206,7 @@ int __kvm_set_memory_region(struct kvm *kvm, return -EINVAL; if ((mem->userspace_addr != old->userspace_addr) || (npages != old->npages) || - ((mem->flags ^ old->flags) & KVM_MEM_READONLY)) + ((mem->flags ^ old->flags) & (KVM_MEM_READONLY | KVM_MEM_GUEST_MEMFD))) return -EINVAL; if (base_gfn != old->base_gfn) -- Gitee From 7cbb9c5c22a54b79f7468d3f9eba2deeb703566e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 1 Dec 2025 18:03:34 -0800 Subject: [PATCH 03/78] KVM: Harden and prepare for modifying existing guest_memfd memslots ANBZ: #40036 commit af62fe2494da84eb01752282c8228c9bb3fe9f67 upstream. Unbind guest_memfd memslots if KVM commits a MOVE or FLAGS_ONLY memslot change to harden against use-after-free, and to prepare for eventually supporting dirty logging on guest_memfd memslots, at which point FLAGS_ONLY changes will be expected/supported. Add two separate WARNs, once to yell if a guest_memfd memslot is moved (which KVM is never expected to allow/support), and again if the unbind() is triggered, to help detect uAPI goofs prior to deliberately allowing FLAGS_ONLY changes. Intel-SIG: commit af62fe2494da KVM: Harden and prepare for modifying existing guest_memfd memslots Link: https://patch.msgid.link/20251202020334.1171351-3-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- virt/kvm/kvm_main.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 6d035f4d85a1..18f832bf3a27 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1871,6 +1871,12 @@ static void kvm_commit_memory_region(struct kvm *kvm, kvm_free_memslot(kvm, old); break; case KVM_MR_MOVE: + /* + * Moving a guest_memfd memslot isn't supported, and will never + * be supported. + */ + WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD); + fallthrough; case KVM_MR_FLAGS_ONLY: /* * Free the dirty bitmap as needed; the below check encompasses @@ -1879,6 +1885,15 @@ static void kvm_commit_memory_region(struct kvm *kvm, if (old->dirty_bitmap && !new->dirty_bitmap) kvm_destroy_dirty_bitmap(old); + /* + * Unbind the guest_memfd instance as needed; the @new slot has + * already created its own binding. TODO: Drop the WARN when + * dirty logging guest_memfd memslots is supported. Until then, + * flags-only changes on guest_memfd slots should be impossible. + */ + if (WARN_ON_ONCE(old->flags & KVM_MEM_GUEST_MEMFD)) + kvm_gmem_unbind(old); + /* * The final quirk. Free the detached, old slot, but only its * memory, not any metadata. Metadata, including arch specific -- Gitee From 43d95e73898b0a457555ab76c7d1a8301a3175b0 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Fri, 12 Dec 2025 21:50:51 +0800 Subject: [PATCH 04/78] KVM: x86: Don't read guest CR3 when doing async pf while the MMU is direct ANBZ: #40036 commit 57a7b47ab30f6201769988392dc8b1c0822a3369 upstream. Don't read guest CR3 in kvm_arch_setup_async_pf() if the MMU is direct and use INVALID_GPA instead. When KVM tries to perform the host-only async page fault for the shared memory of TDX guests, the following WARNING is triggered: WARNING: CPU: 1 PID: 90922 at arch/x86/kvm/vmx/main.c:483 vt_cache_reg+0x16/0x20 Call Trace: __kvm_mmu_faultin_pfn kvm_mmu_faultin_pfn kvm_tdp_page_fault kvm_mmu_do_page_fault kvm_mmu_page_fault tdx_handle_ept_violation This WARNING is triggered when calling kvm_mmu_get_guest_pgd() to cache the guest CR3 in kvm_arch_setup_async_pf() for later use in kvm_arch_async_page_ready() to determine if it's possible to fix the page fault in the current vCPU context to save one VM exit. However, when guest state is protected, KVM cannot read the guest CR3. Since protected guests aren't compatible with shadow paging, i.e, they must use direct MMU, avoid calling kvm_mmu_get_guest_pgd() to read guest CR3 when the MMU is direct and use INVALID_GPA instead. Note that for protected guests mmu->root_role.direct is always true, so that kvm_mmu_get_guest_pgd() in kvm_arch_async_page_ready() won't be reached. Intel-SIG: commit 57a7b47ab30f KVM: x86: Don't read guest CR3 when doing async pf while the MMU is direct Reported-by: Farrah Chen Suggested-by: Sean Christopherson Signed-off-by: Xiaoyao Li Link: https://patch.msgid.link/20251212135051.2155280-1-xiaoyao.li@intel.com [sean: explicitly cast to "unsigned long" to make 32-bit builds happy] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/mmu/mmu.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b117e522312a..205269c1c581 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4572,7 +4572,10 @@ static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, arch.gfn = fault->gfn; arch.error_code = fault->error_code; arch.direct_map = vcpu->arch.mmu->root_role.direct; - arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu); + if (arch.direct_map) + arch.cr3 = (unsigned long)INVALID_GPA; + else + arch.cr3 = kvm_mmu_get_guest_pgd(vcpu, vcpu->arch.mmu); return kvm_setup_async_pf(vcpu, fault->addr, kvm_vcpu_gfn_to_hva(vcpu, fault->gfn), &arch); -- Gitee From 9c1bcf8ffc68de56d5c144371469a69c41835b39 Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 8 Jan 2026 15:46:18 -0600 Subject: [PATCH 05/78] KVM: guest_memfd: Remove partial hugepage handling from kvm_gmem_populate() ANBZ: #40036 commit 6538b6221cc2feda415ca1946e66a5ef02dc6a0a upstream. kvm_gmem_populate(), and the associated post-populate callbacks, have some limited support for dealing with guests backed by hugepages by passing the order information along to each post-populate callback and iterating through the pages passed to kvm_gmem_populate() in hugepage-chunks. However, guest_memfd doesn't yet support hugepages, and in most cases additional changes in the kvm_gmem_populate() path would also be needed to actually allow for this functionality. This makes the existing code unnecessarily complex, and makes changes difficult to work through upstream due to theoretical impacts on hugepage support that can't be considered properly without an actual hugepage implementation to reference. So for now, remove what's there so changes for things like in-place conversion can be implemented/reviewed more efficiently. Intel-SIG: commit 6538b6221cc2 KVM: guest_memfd: Remove partial hugepage handling from kvm_gmem_populate() Suggested-by: Vishal Annapurve Co-developed-by: Vishal Annapurve Signed-off-by: Vishal Annapurve Tested-by: Vishal Annapurve Tested-by: Kai Huang Signed-off-by: Michael Roth Tested-by: Yan Zhao Reviewed-by: Yan Zhao Link: https://patch.msgid.link/20260108214622.1084057-3-michael.roth@amd.com [sean: check for !IS_ERR() before checking folio_order()] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 94 ++++++++++++++++------------------------ arch/x86/kvm/vmx/tdx.c | 2 +- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 30 +++++++------ 4 files changed, 56 insertions(+), 72 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 259af9317b03..f3365d82478e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2374,67 +2374,53 @@ struct sev_gmem_populate_args { int fw_error; }; -static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn_start, kvm_pfn_t pfn, - void __user *src, int order, void *opaque) +static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + void __user *src, void *opaque) { struct sev_gmem_populate_args *sev_populate_args = opaque; + struct sev_data_snp_launch_update fw_args = {0}; struct kvm_sev_info *sev = to_kvm_sev_info(kvm); - int n_private = 0, ret, i; - int npages = (1 << order); - gfn_t gfn; + bool assigned = false; + int level; + int ret; if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src)) return -EINVAL; - for (gfn = gfn_start, i = 0; gfn < gfn_start + npages; gfn++, i++) { - struct sev_data_snp_launch_update fw_args = {0}; - bool assigned = false; - int level; - - ret = snp_lookup_rmpentry((u64)pfn + i, &assigned, &level); - if (ret || assigned) { - pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n", - __func__, gfn, ret, assigned); - ret = ret ? -EINVAL : -EEXIST; - goto err; - } + ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level); + if (ret || assigned) { + pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n", + __func__, gfn, ret, assigned); + ret = ret ? -EINVAL : -EEXIST; + goto out; + } - if (src) { - void *vaddr = kmap_local_pfn(pfn + i); + if (src) { + void *vaddr = kmap_local_pfn(pfn); - if (copy_from_user(vaddr, src + i * PAGE_SIZE, PAGE_SIZE)) { - kunmap_local(vaddr); - ret = -EFAULT; - goto err; - } + if (copy_from_user(vaddr, src, PAGE_SIZE)) { kunmap_local(vaddr); + ret = -EFAULT; + goto out; } - - ret = rmp_make_private(pfn + i, gfn << PAGE_SHIFT, PG_LEVEL_4K, - sev_get_asid(kvm), true); - if (ret) - goto err; - - n_private++; - - fw_args.gctx_paddr = __psp_pa(sev->snp_context); - fw_args.address = __sme_set(pfn_to_hpa(pfn + i)); - fw_args.page_size = PG_LEVEL_TO_RMP(PG_LEVEL_4K); - fw_args.page_type = sev_populate_args->type; - - ret = __sev_issue_cmd(sev_populate_args->sev_fd, SEV_CMD_SNP_LAUNCH_UPDATE, - &fw_args, &sev_populate_args->fw_error); - if (ret) - goto fw_err; + kunmap_local(vaddr); } - return 0; + ret = rmp_make_private(pfn, gfn << PAGE_SHIFT, PG_LEVEL_4K, + sev_get_asid(kvm), true); + if (ret) + goto out; + + fw_args.gctx_paddr = __psp_pa(sev->snp_context); + fw_args.address = __sme_set(pfn_to_hpa(pfn)); + fw_args.page_size = PG_LEVEL_TO_RMP(PG_LEVEL_4K); + fw_args.page_type = sev_populate_args->type; -fw_err: + ret = __sev_issue_cmd(sev_populate_args->sev_fd, SEV_CMD_SNP_LAUNCH_UPDATE, + &fw_args, &sev_populate_args->fw_error); /* * If the firmware command failed handle the reclaim and cleanup of that - * PFN specially vs. prior pages which can be cleaned up below without - * needing to reclaim in advance. + * PFN before reporting an error. * * Additionally, when invalid CPUID function entries are detected, * firmware writes the expected values into the page and leaves it @@ -2444,26 +2430,20 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn_start, kvm_pfn_t pf * information to provide information on which CPUID leaves/fields * failed CPUID validation. */ - if (!snp_page_reclaim(kvm, pfn + i) && + if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { - void *vaddr = kmap_local_pfn(pfn + i); + void *vaddr = kmap_local_pfn(pfn); - if (copy_to_user(src + i * PAGE_SIZE, vaddr, PAGE_SIZE)) + if (copy_to_user(src, vaddr, PAGE_SIZE)) pr_debug("Failed to write CPUID page back to userspace\n"); kunmap_local(vaddr); } - /* pfn + i is hypervisor-owned now, so skip below cleanup for it. */ - n_private--; - -err: - pr_debug("%s: exiting with error ret %d (fw_error %d), restoring %d gmem PFNs to shared.\n", - __func__, ret, sev_populate_args->fw_error, n_private); - for (i = 0; i < n_private; i++) - kvm_rmp_make_shared(kvm, pfn + i, PG_LEVEL_4K); - +out: + pr_debug("%s: exiting with return code %d (fw_error %d)\n", + __func__, ret, sev_populate_args->fw_error); return ret; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index d6358630f77e..5db26d4035c0 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3144,7 +3144,7 @@ struct tdx_gmem_post_populate_arg { }; static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - void __user *src, int order, void *_arg) + void __user *src, void *_arg) { struct tdx_gmem_post_populate_arg *arg = _arg; struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index e7bce38e9892..8c1de8c324de 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2633,7 +2633,7 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord * Returns the number of pages that were populated. */ typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - void __user *src, int order, void *opaque); + void __user *src, void *opaque); long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, kvm_gmem_populate_cb post_populate, void *opaque); diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 5e3660c9fbc5..68f3b10bc4c2 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -151,6 +151,15 @@ static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index) mapping_gfp_mask(inode->i_mapping), policy); mpol_cond_put(policy); + /* + * External interfaces like kvm_gmem_get_pfn() support dealing + * with hugepages to a degree, but internally, guest_memfd currently + * assumes that all folios are order-0 and handling would need + * to be updated for anything otherwise (e.g. page-clearing + * operations). + */ + WARN_ON_ONCE(!IS_ERR(folio) && folio_order(folio)); + return folio; } @@ -829,7 +838,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long struct kvm_memory_slot *slot; void __user *p; - int ret = 0, max_order; + int ret = 0; long i; lockdep_assert_held(&kvm->slots_lock); @@ -847,7 +856,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long filemap_invalidate_lock(file->f_mapping); npages = min_t(ulong, slot->npages - (start_gfn - slot->base_gfn), npages); - for (i = 0; i < npages; i += (1 << max_order)) { + for (i = 0; i < npages; i++) { struct folio *folio; gfn_t gfn = start_gfn + i; pgoff_t index = kvm_gmem_get_index(slot, gfn); @@ -859,7 +868,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long break; } - folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, &is_prepared, &max_order); + folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, &is_prepared, NULL); if (IS_ERR(folio)) { ret = PTR_ERR(folio); break; @@ -873,20 +882,15 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long } folio_unlock(folio); - WARN_ON(!IS_ALIGNED(gfn, 1 << max_order) || - (npages - i) < (1 << max_order)); ret = -EINVAL; - while (!kvm_range_has_memory_attributes(kvm, gfn, gfn + (1 << max_order), - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) { - if (!max_order) - goto put_folio_and_exit; - max_order--; - } + if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + KVM_MEMORY_ATTRIBUTE_PRIVATE)) + goto put_folio_and_exit; p = src ? src + i * PAGE_SIZE : NULL; - ret = post_populate(kvm, gfn, pfn, p, max_order, opaque); + ret = post_populate(kvm, gfn, pfn, p, opaque); if (!ret) kvm_gmem_mark_prepared(folio); -- Gitee From 5b37533d2b4073a7729c4770ffc801135e570c7c Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 8 Jan 2026 15:46:19 -0600 Subject: [PATCH 06/78] KVM: guest_memfd: Remove preparation tracking ANBZ: #40036 commit 8622ef05709fbc4903f54cdf1ac8c3725e479dd8 upstream. guest_memfd currently uses the folio uptodate flag to track: 1) whether or not a page has been cleared before initial usage 2) whether or not the architecture hooks have been issued to put the page in a private state as defined by the architecture In practice, (2) is only actually being tracked for SEV-SNP VMs, and there do not seem to be any plans/reasons that would suggest this will change in the future, so this additional tracking/complexity is not really providing any general benefit to guest_memfd users. On the other hand, future plans around in-place conversion and hugepage support will make the burden of tracking this information within guest_memfd even more complex. With in-place conversion and hugepage support, the plan is to use the per-folio uptodate flag purely to track the initial clearing of folios, whereas conversion operations could trigger multiple transitions between 'prepared' and 'unprepared' and thus need separate tracking. Since preparation generally happens during fault time, i.e. on the "read-side" of any VM-wide locks that might protect state tracked by guest_memfd, supporting concurrent handling of page faults would likely require more complex locking schemes if the "preparedness" state were tracked by guest_memfd, i.e. if it needs to be updated as part of handling the fault. Instead of keeping this current/future complexity within guest_memfd for what is essentially just SEV-SNP, just drop the tracking for (2) and have the arch-specific preparation hooks get triggered unconditionally on every fault so the arch-specific hooks can check the preparation state directly and decide whether or not a folio still needs additional preparation. In the case of SEV-SNP, the preparation state is already checked again via the preparation hooks to avoid double-preparation, so nothing extra needs to be done to update the handling of things there. Intel-SIG: commit 8622ef05709f KVM: guest_memfd: Remove preparation tracking Reviewed-by: Vishal Annapurve Tested-by: Vishal Annapurve Reviewed-by: Pankaj Gupta Tested-by: Kai Huang Signed-off-by: Michael Roth Link: https://patch.msgid.link/20260108214622.1084057-4-michael.roth@amd.com [sean: massage changelog] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- virt/kvm/guest_memfd.c | 45 +++++++++++------------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 68f3b10bc4c2..f3f13fdc3987 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -76,11 +76,6 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo return 0; } -static inline void kvm_gmem_mark_prepared(struct folio *folio) -{ - folio_mark_uptodate(folio); -} - /* * Process @folio, which contains @gfn, so that the guest can use it. * The folio must be locked and the gfn must be contained in @slot. @@ -90,13 +85,7 @@ static inline void kvm_gmem_mark_prepared(struct folio *folio) static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, struct folio *folio) { - unsigned long nr_pages, i; pgoff_t index; - int r; - - nr_pages = folio_nr_pages(folio); - for (i = 0; i < nr_pages; i++) - clear_highpage(folio_page(folio, i)); /* * Preparing huge folios should always be safe, since it should @@ -114,11 +103,7 @@ static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, folio_nr_pages(folio))); index = kvm_gmem_get_index(slot, gfn); index = ALIGN_DOWN(index, folio_nr_pages(folio)); - r = __kvm_gmem_prepare_folio(kvm, slot, index, folio); - if (!r) - kvm_gmem_mark_prepared(folio); - - return r; + return __kvm_gmem_prepare_folio(kvm, slot, index, folio); } /* @@ -430,7 +415,7 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (!folio_test_uptodate(folio)) { clear_highpage(folio_page(folio, 0)); - kvm_gmem_mark_prepared(folio); + folio_mark_uptodate(folio); } vmf->page = folio_file_page(folio, vmf->pgoff); @@ -767,7 +752,7 @@ void kvm_gmem_unbind(struct kvm_memory_slot *slot) static struct folio *__kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot, pgoff_t index, kvm_pfn_t *pfn, - bool *is_prepared, int *max_order) + int *max_order) { struct gmem_file *f = file->private_data; struct folio *folio; @@ -796,7 +781,6 @@ static struct folio *__kvm_gmem_get_pfn(struct file *file, if (max_order) *max_order = 0; - *is_prepared = folio_test_uptodate(folio); return folio; } @@ -806,19 +790,22 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; - bool is_prepared = false; int r = 0; CLASS(gmem_get_file, file)(slot); if (!file) return -EFAULT; - folio = __kvm_gmem_get_pfn(file, slot, index, pfn, &is_prepared, max_order); + folio = __kvm_gmem_get_pfn(file, slot, index, pfn, max_order); if (IS_ERR(folio)) return PTR_ERR(folio); - if (!is_prepared) - r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); + if (!folio_test_uptodate(folio)) { + clear_highpage(folio_page(folio, 0)); + folio_mark_uptodate(folio); + } + + r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); folio_unlock(folio); @@ -860,7 +847,6 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long struct folio *folio; gfn_t gfn = start_gfn + i; pgoff_t index = kvm_gmem_get_index(slot, gfn); - bool is_prepared = false; kvm_pfn_t pfn; if (signal_pending(current)) { @@ -868,19 +854,12 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long break; } - folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, &is_prepared, NULL); + folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, NULL); if (IS_ERR(folio)) { ret = PTR_ERR(folio); break; } - if (is_prepared) { - folio_unlock(folio); - folio_put(folio); - ret = -EEXIST; - break; - } - folio_unlock(folio); ret = -EINVAL; @@ -892,7 +871,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long p = src ? src + i * PAGE_SIZE : NULL; ret = post_populate(kvm, gfn, pfn, p, opaque); if (!ret) - kvm_gmem_mark_prepared(folio); + folio_mark_uptodate(folio); put_folio_and_exit: folio_put(folio); -- Gitee From 9964e909a06b63b2fd02439d33eaa293367d482d Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 8 Jan 2026 15:46:21 -0600 Subject: [PATCH 07/78] KVM: TDX: Document alignment requirements for KVM_TDX_INIT_MEM_REGION ANBZ: #40036 commit 189fd1b059a9c7ed22750bc8a4a1182c58ccf138 upstream. Since it was never possible to use a non-PAGE_SIZE-aligned @source_addr, go ahead and document this as a requirement. This is in preparation for enforcing page-aligned @source_addr for all architectures in guest_memfd. Intel-SIG: commit 189fd1b059a9 KVM: TDX: Document alignment requirements for KVM_TDX_INIT_MEM_REGION Reviewed-by: Vishal Annapurve Tested-by: Kai Huang Signed-off-by: Michael Roth Reviewed-by: Yan Zhao Link: https://patch.msgid.link/20260108214622.1084057-6-michael.roth@amd.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/virt/kvm/x86/intel-tdx.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/x86/intel-tdx.rst b/Documentation/virt/kvm/x86/intel-tdx.rst index 5efac62c92c7..6a222e9d0954 100644 --- a/Documentation/virt/kvm/x86/intel-tdx.rst +++ b/Documentation/virt/kvm/x86/intel-tdx.rst @@ -156,7 +156,7 @@ KVM_TDX_INIT_MEM_REGION :Returns: 0 on success, <0 on error Initialize @nr_pages TDX guest private memory starting from @gpa with userspace -provided data from @source_addr. +provided data from @source_addr. @source_addr must be PAGE_SIZE-aligned. Note, before calling this sub command, memory attribute of the range [gpa, gpa + nr_pages] needs to be private. Userspace can use -- Gitee From 3278b295b1d37818ce126ade6bbe5d7d7961eefd Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 8 Jan 2026 15:46:20 -0600 Subject: [PATCH 08/78] KVM: SEV: Document/enforce page-alignment for KVM_SEV_SNP_LAUNCH_UPDATE ANBZ: #40036 commit dcbcc2323c806b55939d765c13d0728421756017 upsteam. In the past, KVM_SEV_SNP_LAUNCH_UPDATE accepted a non-page-aligned 'uaddr' parameter to copy data from, but continuing to support this with new functionality like in-place conversion and hugepages in the pipeline has proven to be more trouble than it is worth, since there are no known users that have been identified who use a non-page-aligned 'uaddr' parameter. Rather than locking guest_memfd into continuing to support this, go ahead and document page-alignment as a requirement and begin enforcing this in the handling function. Intel-SIG: commit dcbcc2323c8 KVM: SEV: Document/enforce page-alignment for KVM_SEV_SNP_LAUNCH_UPDATE Reviewed-by: Vishal Annapurve Tested-by: Kai Huang Signed-off-by: Michael Roth Link: https://patch.msgid.link/20260108214622.1084057-5-michael.roth@amd.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/virt/kvm/x86/amd-memory-encryption.rst | 2 +- arch/x86/kvm/svm/sev.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/virt/kvm/x86/amd-memory-encryption.rst b/Documentation/virt/kvm/x86/amd-memory-encryption.rst index 1ddb6a86ce7f..5a88d0197cb3 100644 --- a/Documentation/virt/kvm/x86/amd-memory-encryption.rst +++ b/Documentation/virt/kvm/x86/amd-memory-encryption.rst @@ -523,7 +523,7 @@ Returns: 0 on success, < 0 on error, -EAGAIN if caller should retry struct kvm_sev_snp_launch_update { __u64 gfn_start; /* Guest page number to load/encrypt data into. */ - __u64 uaddr; /* Userspace address of data to be loaded/encrypted. */ + __u64 uaddr; /* 4k-aligned address of data to be loaded/encrypted. */ __u64 len; /* 4k-aligned length in bytes to copy into guest memory.*/ __u8 type; /* The type of the guest pages being initialized. */ __u8 pad0; diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index f3365d82478e..b2bebdcbd0a1 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2474,6 +2474,11 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID)) return -EINVAL; + src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr); + + if (!PAGE_ALIGNED(src)) + return -EINVAL; + npages = params.len / PAGE_SIZE; /* @@ -2505,7 +2510,6 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) sev_populate_args.sev_fd = argp->sev_fd; sev_populate_args.type = params.type; - src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr); count = kvm_gmem_populate(kvm, params.gfn_start, src, npages, sev_gmem_post_populate, &sev_populate_args); -- Gitee From 4579adc721f7412752b831411f8727d388b2644f Mon Sep 17 00:00:00 2001 From: Michael Roth Date: Thu, 8 Jan 2026 15:46:22 -0600 Subject: [PATCH 09/78] KVM: guest_memfd: GUP source pages prior to populating guest memory ANBZ: #40036 commit 2a62345b30529e488beb6a1220577b3495933724 upstream. Currently the post-populate callbacks handle copying source pages into private GPA ranges backed by guest_memfd, where kvm_gmem_populate() acquires the filemap invalidate lock, then calls a post-populate callback which may issue a get_user_pages() on the source pages prior to copying them into the private GPA (e.g. TDX). This will not be compatible with in-place conversion, where the userspace page fault path will attempt to acquire the filemap invalidate lock while holding the mm->mmap_lock, leading to a potential ABBA deadlock. Address this by hoisting the GUP above the filemap invalidate lock so that these page faults path can be taken early, prior to acquiring the filemap invalidate lock. It's not currently clear whether this issue is reachable with the current implementation of guest_memfd, which doesn't support in-place conversion, however it does provide a consistent mechanism to provide stable source/target PFNs to callbacks rather than punting to vendor-specific code, which allows for more commonality across architectures, which may be worthwhile even without in-place conversion. As part of this change, also begin enforcing that the 'src' argument to kvm_gmem_populate() must be page-aligned, as this greatly reduces the complexity around how the post-populate callbacks are implemented, and since no current in-tree users support using a non-page-aligned 'src' argument. Intel-SIG: commit 2a62345b3052 KVM: guest_memfd: GUP source pages prior to populating guest memory Suggested-by: Sean Christopherson Co-developed-by: Sean Christopherson Co-developed-by: Vishal Annapurve Signed-off-by: Vishal Annapurve Tested-by: Vishal Annapurve Tested-by: Kai Huang Signed-off-by: Michael Roth Tested-by: Yan Zhao Reviewed-by: Yan Zhao Link: https://patch.msgid.link/20260108214622.1084057-7-michael.roth@amd.com [sean: avoid local "p" variable] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 33 ++++++++-------- arch/x86/kvm/vmx/tdx.c | 16 ++------ include/linux/kvm_host.h | 4 +- virt/kvm/guest_memfd.c | 83 +++++++++++++++++++++++++++------------- 4 files changed, 78 insertions(+), 58 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index b2bebdcbd0a1..afd163cffe76 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2375,7 +2375,7 @@ struct sev_gmem_populate_args { }; static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - void __user *src, void *opaque) + struct page *src_page, void *opaque) { struct sev_gmem_populate_args *sev_populate_args = opaque; struct sev_data_snp_launch_update fw_args = {0}; @@ -2384,7 +2384,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int level; int ret; - if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src)) + if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src_page)) return -EINVAL; ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level); @@ -2395,15 +2395,14 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, goto out; } - if (src) { - void *vaddr = kmap_local_pfn(pfn); + if (src_page) { + void *src_vaddr = kmap_local_page(src_page); + void *dst_vaddr = kmap_local_pfn(pfn); - if (copy_from_user(vaddr, src, PAGE_SIZE)) { - kunmap_local(vaddr); - ret = -EFAULT; - goto out; - } - kunmap_local(vaddr); + memcpy(dst_vaddr, src_vaddr, PAGE_SIZE); + + kunmap_local(src_vaddr); + kunmap_local(dst_vaddr); } ret = rmp_make_private(pfn, gfn << PAGE_SHIFT, PG_LEVEL_4K, @@ -2433,17 +2432,19 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (ret && !snp_page_reclaim(kvm, pfn) && sev_populate_args->type == KVM_SEV_SNP_PAGE_TYPE_CPUID && sev_populate_args->fw_error == SEV_RET_INVALID_PARAM) { - void *vaddr = kmap_local_pfn(pfn); + void *src_vaddr = kmap_local_page(src_page); + void *dst_vaddr = kmap_local_pfn(pfn); - if (copy_to_user(src, vaddr, PAGE_SIZE)) - pr_debug("Failed to write CPUID page back to userspace\n"); + memcpy(src_vaddr, dst_vaddr, PAGE_SIZE); - kunmap_local(vaddr); + kunmap_local(src_vaddr); + kunmap_local(dst_vaddr); } out: - pr_debug("%s: exiting with return code %d (fw_error %d)\n", - __func__, ret, sev_populate_args->fw_error); + if (ret) + pr_debug("%s: error updating GFN %llx, return code %d (fw_error %d)\n", + __func__, gfn, ret, sev_populate_args->fw_error); return ret; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 5db26d4035c0..48f621b2992c 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3144,34 +3144,24 @@ struct tdx_gmem_post_populate_arg { }; static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - void __user *src, void *_arg) + struct page *src_page, void *_arg) { struct tdx_gmem_post_populate_arg *arg = _arg; struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm); u64 err, entry, level_state; gpa_t gpa = gfn_to_gpa(gfn); - struct page *src_page; int ret, i; if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - /* - * Get the source page if it has been faulted in. Return failure if the - * source page has been swapped out or unmapped in primary memory. - */ - ret = get_user_pages_fast((unsigned long)src, 1, 0, &src_page); - if (ret < 0) - return ret; - if (ret != 1) - return -ENOMEM; + if (!src_page) + return -EOPNOTSUPP; kvm_tdx->page_add_src = src_page; ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; - put_page(src_page); - if (ret || !(arg->flags & KVM_TDX_MEASURE_MEMORY_REGION)) return ret; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 8c1de8c324de..2c8d50d206ca 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2618,7 +2618,7 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord * @gfn: starting GFN to be populated * @src: userspace-provided buffer containing data to copy into GFN range * (passed to @post_populate, and incremented on each iteration - * if not NULL) + * if not NULL). Must be page-aligned. * @npages: number of pages to copy from userspace-buffer * @post_populate: callback to issue for each gmem page that backs the GPA * range @@ -2633,7 +2633,7 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord * Returns the number of pages that were populated. */ typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - void __user *src, void *opaque); + struct page *page, void *opaque); long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, kvm_gmem_populate_cb post_populate, void *opaque); diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index f3f13fdc3987..3f0216045e42 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -819,12 +819,48 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE + +static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, + struct file *file, gfn_t gfn, struct page *src_page, + kvm_gmem_populate_cb post_populate, void *opaque) +{ + pgoff_t index = kvm_gmem_get_index(slot, gfn); + struct folio *folio; + kvm_pfn_t pfn; + int ret; + + filemap_invalidate_lock(file->f_mapping); + + folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, NULL); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_unlock; + } + + folio_unlock(folio); + + if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, + KVM_MEMORY_ATTRIBUTE_PRIVATE, + KVM_MEMORY_ATTRIBUTE_PRIVATE)) { + ret = -EINVAL; + goto out_put_folio; + } + + ret = post_populate(kvm, gfn, pfn, src_page, opaque); + if (!ret) + folio_mark_uptodate(folio); + +out_put_folio: + folio_put(folio); +out_unlock: + filemap_invalidate_unlock(file->f_mapping); + return ret; +} + long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages, kvm_gmem_populate_cb post_populate, void *opaque) { struct kvm_memory_slot *slot; - void __user *p; - int ret = 0; long i; @@ -832,6 +868,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long if (npages < 0) return -EINVAL; + if (WARN_ON_ONCE(!PAGE_ALIGNED(src))) + return -EINVAL; + slot = gfn_to_memslot(kvm, start_gfn); if (!kvm_slot_has_gmem(slot)) return -EINVAL; @@ -840,47 +879,37 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long if (!file) return -EFAULT; - filemap_invalidate_lock(file->f_mapping); - npages = min_t(ulong, slot->npages - (start_gfn - slot->base_gfn), npages); for (i = 0; i < npages; i++) { - struct folio *folio; - gfn_t gfn = start_gfn + i; - pgoff_t index = kvm_gmem_get_index(slot, gfn); - kvm_pfn_t pfn; + struct page *src_page = NULL; if (signal_pending(current)) { ret = -EINTR; break; } - folio = __kvm_gmem_get_pfn(file, slot, index, &pfn, NULL); - if (IS_ERR(folio)) { - ret = PTR_ERR(folio); - break; + if (src) { + unsigned long uaddr = (unsigned long)src + i * PAGE_SIZE; + + ret = get_user_pages_fast(uaddr, 1, 0, &src_page); + if (ret < 0) + break; + if (ret != 1) { + ret = -ENOMEM; + break; + } } - folio_unlock(folio); + ret = __kvm_gmem_populate(kvm, slot, file, start_gfn + i, src_page, + post_populate, opaque); - ret = -EINVAL; - if (!kvm_range_has_memory_attributes(kvm, gfn, gfn + 1, - KVM_MEMORY_ATTRIBUTE_PRIVATE, - KVM_MEMORY_ATTRIBUTE_PRIVATE)) - goto put_folio_and_exit; + if (src_page) + put_page(src_page); - p = src ? src + i * PAGE_SIZE : NULL; - ret = post_populate(kvm, gfn, pfn, p, opaque); - if (!ret) - folio_mark_uptodate(folio); - -put_folio_and_exit: - folio_put(folio); if (ret) break; } - filemap_invalidate_unlock(file->f_mapping); - return ret && !i ? ret : i; } EXPORT_SYMBOL_GPL(kvm_gmem_populate); -- Gitee From 12a09ba93a78f9839ee6b6e9d3ae3f8d37fcad0e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 30 Jun 2026 14:37:10 -0700 Subject: [PATCH 10/78] KVM: SEV: Explicitly disallow NULL user address for SNP_LAUNCH_UPDATE ANBZ: #40036 commit 2abe1ff20151f47a0c26bc2275e42718ae40553b upstream. Explicitly reject a NULL userspace virtual address for the source page of SNP_LAUNCH_UPDATE instead of relying on the post-populate callback to do the check, and don't WARN on failure, as the scenario is blatantly user- triggerable, as reported by Sashiko. Waiting until post-populate to check the address "works", but makes it unnecessarily difficult to see that KVM's ABI is to disallow a NULL source page for non-ZERO pages. Note, several existing VMMs pass a valid userspace address for the ZERO case, i.e. KVM can't *require* the userspace address to be NULL for ZERO pages, at least not without breaking userspace. Intel-SIG: commit 2abe1ff20151 KVM: SEV: Explicitly disallow NULL user address for SNP_LAUNCH_UPDATE Fixes: dee5a47cc7a4 ("KVM: SEV: Add KVM_SEV_SNP_LAUNCH_UPDATE command") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260611125849.9ED631F00893@smtp.kernel.org Signed-off-by: Joerg Roedel Co-developed-by: Sean Christopherson Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260630213711.479692-2-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index afd163cffe76..2078e683c11c 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2384,9 +2384,6 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int level; int ret; - if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src_page)) - return -EINVAL; - ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level); if (ret || assigned) { pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n", @@ -2475,10 +2472,12 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID)) return -EINVAL; - src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr); - - if (!PAGE_ALIGNED(src)) + if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) + src = NULL; + else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) return -EINVAL; + else + src = u64_to_user_ptr(params.uaddr); npages = params.len / PAGE_SIZE; -- Gitee From f52668a39a2cbf8895cbf5227954f4bac225fe23 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Tue, 3 Mar 2026 09:51:58 +1300 Subject: [PATCH 11/78] KVM: selftests: Increase 'maxnode' for guest_memfd tests ANBZ: #40036 commit cf534a09fb621b0aa875613e3cd88aee336e16d7 upstream. Increase 'maxnode' when using 'get_mempolicy' syscall in guest_memfd mmap and NUMA policy tests to fix a failure on one Intel GNR platform. On a CXL-capable platform, the memory affinity of CXL memory regions may not be covered by the SRAT. Since each CXL memory region is enumerated via a CFMWS table, at early boot the kernel parses all CFMWS tables to detect all CXL memory regions and assigns a 'faked' NUMA node for each of them, starting from the highest NUMA node ID enumerated via the SRAT. This increases the 'nr_node_ids'. E.g., on the aforementioned Intel GNR platform which has 4 NUMA nodes and 18 CFMWS tables, it increases to 22. This results in the 'get_mempolicy' syscall failure on that platform, because currently 'maxnode' is hard-coded to 8 but the 'get_mempolicy' syscall requires the 'maxnode' to be not smaller than the 'nr_node_ids'. Increase the 'maxnode' to the number of bits of 'nodemask', which is 'unsigned long', to fix this. This may not cover all systems. Perhaps a better way is to always set the 'nodemask' and 'maxnode' based on the actual maximum NUMA node ID on the system, but for now just do the simple way. Intel-SIG: commit cf534a09fb62 KVM: selftests: Increase 'maxnode' for guest_memfd tests Reported-by: Yi Lai Closes: https://bugzilla.kernel.org/show_bug.cgi?id=221014 Closes: https://lore.kernel.org/all/bug-221014-28872@https.bugzilla.kernel.org%2F Signed-off-by: Kai Huang Reviewed-by: Yuan Yao Link: https://patch.msgid.link/20260302205158.178058-1-kai.huang@intel.com Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- tools/testing/selftests/kvm/guest_memfd_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/guest_memfd_test.c b/tools/testing/selftests/kvm/guest_memfd_test.c index c87396980f21..28a020201320 100644 --- a/tools/testing/selftests/kvm/guest_memfd_test.c +++ b/tools/testing/selftests/kvm/guest_memfd_test.c @@ -82,7 +82,7 @@ static void test_mbind(int fd, size_t total_size) { const unsigned long nodemask_0 = 1; /* nid: 0 */ unsigned long nodemask = 0; - unsigned long maxnode = 8; + unsigned long maxnode = BITS_PER_TYPE(nodemask); int policy; char *mem; int ret; -- Gitee From 89c14535ca1bf162225df490d3ee30483b6c4b02 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Fri, 2 May 2025 16:12:05 +0200 Subject: [PATCH 12/78] modpost: Use for() loop ANBZ: #40036 commit 6b91ff002c67b2502009f99115c4b7fe5b7b8248 upstream. Slight cleanup by using a for() loop instead of while(). This makes it clearer what is the iteration and what is the actual work done. Intel-SIG: commit 6b91ff002c67 modpost: Use for() loop Signed-off-by: Peter Zijlstra Signed-off-by: Masahiro Yamada [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- scripts/mod/modpost.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 423e284a57eb..5316de21ae40 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1655,12 +1655,10 @@ static void read_symbols(const char *modname) license = get_next_modinfo(&info, "license", license); } - namespace = get_modinfo(&info, "import_ns"); - while (namespace) { + for (namespace = get_modinfo(&info, "import_ns"); + namespace; + namespace = get_next_modinfo(&info, "import_ns", namespace)) add_namespace(&mod->imported_namespaces, namespace); - namespace = get_next_modinfo(&info, "import_ns", - namespace); - } if (extra_warn && !get_modinfo(&info, "description")) warn("missing MODULE_DESCRIPTION() in %s\n", modname); -- Gitee From 22833442b16afec8685427545c60e5e71196bdbe Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Thu, 22 May 2025 16:17:20 +0900 Subject: [PATCH 13/78] modpost: check forbidden MODULE_IMPORT_NS("module:") at compile time ANBZ: #40036 commit 9c036cfbb75bb58ecce589f547fdb8af153493a6 upstream. Explicitly adding MODULE_IMPORT_NS("module:...") is not allowed. Currently, this is only checked at run time. That is, when such a module is loaded, an error message like the following is shown: foo: module tries to import module namespace: module:bar Obviously, checking this at compile time improves usability. In such a case, modpost will report the following error at compile time: Intel-SIG: commit 9c036cfbb75b modpost: check forbidden MODULE_IMPORT_NS("module:") at compile time ERROR: modpost: foo: explicitly importing namespace "module:bar" is not allowed. Signed-off-by: Masahiro Yamada [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- scripts/mod/modpost.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 5316de21ae40..91d62dab85b1 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -23,6 +23,8 @@ #include "modpost.h" #include "../../include/linux/license.h" +#define MODULE_NS_PREFIX "module:" + static bool module_enabled; /* Are we using CONFIG_MODVERSIONS? */ static bool modversions; @@ -1657,8 +1659,13 @@ static void read_symbols(const char *modname) for (namespace = get_modinfo(&info, "import_ns"); namespace; - namespace = get_next_modinfo(&info, "import_ns", namespace)) + namespace = get_next_modinfo(&info, "import_ns", namespace)) { + if (strstarts(namespace, MODULE_NS_PREFIX)) + error("%s: explicitly importing namespace \"%s\" is not allowed.\n", + mod->name, namespace); + add_namespace(&mod->imported_namespaces, namespace); + } if (extra_warn && !get_modinfo(&info, "description")) warn("missing MODULE_DESCRIPTION() in %s\n", modname); -- Gitee From 9e2fa458dcbe09acef32c8fa87fba961c4a7f420 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 8 Aug 2025 15:28:47 +0200 Subject: [PATCH 14/78] module: Rename EXPORT_SYMBOL_GPL_FOR_MODULES to EXPORT_SYMBOL_FOR_MODULES ANBZ: #40036 commit 6d3c3ca4c77e93660cce5819bf707f75df03e0c8 upstream. Christoph suggested that the explicit _GPL_ can be dropped from the module namespace export macro, as it's intended for in-tree modules only. It would be possible to restrict it technically, but it was pointed out [2] that some cases of using an out-of-tree build of an in-tree module with the same name are legitimate. But in that case those also have to be GPL anyway so it's unnecessary to spell it out in the macro name. Intel-SIG: commit 6d3c3ca4c77e module: Rename EXPORT_SYMBOL_GPL_FOR_MODULES to EXPORT_SYMBOL_FOR_MODULES Link: https://lore.kernel.org/all/aFleJN_fE-RbSoFD@infradead.org/ [1] Link: https://lore.kernel.org/all/CAK7LNATRkZHwJGpojCnvdiaoDnP%2BaeUXgdey5sb_8muzdWTMkA@mail.gmail.com/ [2] Suggested-by: Christoph Hellwig Reviewed-by: Shivank Garg Acked-by: David Hildenbrand Acked-by: Nicolas Schier Reviewed-by: Daniel Gomez Reviewed-by: Christian Brauner Signed-off-by: Vlastimil Babka Link: https://lore.kernel.org/20250808-export_modules-v4-1-426945bcc5e1@suse.cz Signed-off-by: Christian Brauner [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/core-api/symbol-namespaces.rst | 9 +++++---- arch/x86/virt/hw.c | 0 drivers/vfio/pci/vfio_pci_dmabuf.c | 2 +- fs/anon_inodes.c | 2 +- include/linux/export.h | 1 - 5 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 arch/x86/virt/hw.c diff --git a/Documentation/core-api/symbol-namespaces.rst b/Documentation/core-api/symbol-namespaces.rst index 29875e25e376..01d945b6ef1a 100644 --- a/Documentation/core-api/symbol-namespaces.rst +++ b/Documentation/core-api/symbol-namespaces.rst @@ -87,20 +87,21 @@ unit as preprocessor statement. The above example would then read:: within the corresponding compilation unit before any EXPORT_SYMBOL macro is used. -2.3 Using the EXPORT_SYMBOL_GPL_FOR_MODULES() macro +2.3 Using the EXPORT_SYMBOL_FOR_MODULES() macro =================================================== Symbols exported using this macro are put into a module namespace. This -namespace cannot be imported. +namespace cannot be imported. These exports are GPL-only as they are only +intended for in-tree modules. The macro takes a comma separated list of module names, allowing only those modules to access this symbol. Simple tail-globs are supported. For example: - EXPORT_SYMBOL_GPL_FOR_MODULES(preempt_notifier_inc, "kvm,kvm-*") + EXPORT_SYMBOL_FOR_MODULES(preempt_notifier_inc, "kvm,kvm-*") -will limit usage of this symbol to modules whoes name matches the given +will limit usage of this symbol to modules whose name matches the given patterns. 3. How to use Symbols exported in Namespaces diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c index c907a74fd534..e269842b299c 100644 --- a/drivers/vfio/pci/vfio_pci_dmabuf.c +++ b/drivers/vfio/pci/vfio_pci_dmabuf.c @@ -138,7 +138,7 @@ int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, *phys = priv->phys_vec[0]; return 0; } -EXPORT_SYMBOL_GPL_FOR_MODULES(vfio_pci_dma_buf_iommufd_map, "iommufd"); +EXPORT_SYMBOL_FOR_MODULES(vfio_pci_dma_buf_iommufd_map, "iommufd"); int vfio_pci_core_fill_phys_vec(struct dma_buf_phys_vec *phys_vec, struct vfio_region_dma_range *dma_ranges, diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c index 542fc42d1469..fa666ffe4033 100644 --- a/fs/anon_inodes.c +++ b/fs/anon_inodes.c @@ -85,7 +85,7 @@ struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *n } return inode; } -EXPORT_SYMBOL_GPL_FOR_MODULES(anon_inode_make_secure_inode, "kvm"); +EXPORT_SYMBOL_FOR_MODULES(anon_inode_make_secure_inode, "kvm"); static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, diff --git a/include/linux/export.h b/include/linux/export.h index 55cec36f1958..bcda631a3b0c 100644 --- a/include/linux/export.h +++ b/include/linux/export.h @@ -94,7 +94,6 @@ extern struct module __this_module; #define EXPORT_SYMBOL_NS(sym, ns) __EXPORT_SYMBOL(sym, "", __stringify(ns)) #define EXPORT_SYMBOL_NS_GPL(sym, ns) __EXPORT_SYMBOL(sym, "GPL", __stringify(ns)) -#define EXPORT_SYMBOL_GPL_FOR_MODULES(sym, mods) __EXPORT_SYMBOL(sym, "GPL", "module:" mods) #define EXPORT_SYMBOL_FOR_MODULES(sym, mods) __EXPORT_SYMBOL(sym, "GPL", "module:" mods) #ifdef CONFIG_KVM_EXPORT_SYMBOL -- Gitee From 56c2bc2f0dc8ef4498edc311c72decb223224b04 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 18 Sep 2025 17:33:00 -0700 Subject: [PATCH 15/78] KVM: Export KVM-internal symbols for sub-modules only ANBZ: #40036 commit 20c48920583675e67b3824f147726e0fbda735ce upstream. Rework the vast majority of KVM's exports to expose symbols only to KVM submodules, i.e. to x86's kvm-{amd,intel}.ko and PPC's kvm-{pr,hv}.ko. With few exceptions, KVM's exported APIs are intended (and safe) for KVM- internal usage only. Keep kvm_get_kvm(), kvm_get_kvm_safe(), and kvm_put_kvm() as normal exports, as they are needed by VFIO, and are generally safe for external usage (though ideally even the get/put APIs would be KVM-internal, and VFIO would pin a VM by grabbing a reference to its associated file). Implement a framework in kvm_types.h in anticipation of providing a macro to restrict KVM-specific kernel exports, i.e. to provide symbol exports for KVM if and only if KVM is built as one or more modules. Intel-SIG: commit 20c489205836 KVM: Export KVM-internal symbols for sub-modules only Link: https://lore.kernel.org/r/20250919003303.1355064-3-seanjc@google.com Cc: Nathan Chancellor Signed-off-by: Sean Christopherson Signed-off-by: Paolo Bonzini [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/powerpc/include/asm/Kbuild | 1 - arch/powerpc/include/asm/kvm_types.h | 15 +++ virt/kvm/eventfd.c | 2 +- virt/kvm/guest_memfd.c | 4 +- virt/kvm/kvm_main.c | 132 +++++++++++++-------------- 5 files changed, 84 insertions(+), 70 deletions(-) create mode 100644 arch/powerpc/include/asm/kvm_types.h diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 61a8d5555cd7..71ed33aad766 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -3,7 +3,6 @@ generated-y += syscall_table_32.h generated-y += syscall_table_64.h generated-y += syscall_table_spu.h generic-y += agp.h -generic-y += kvm_types.h generic-y += mcs_spinlock.h generic-y += qrwlock.h generic-y += vtime.h diff --git a/arch/powerpc/include/asm/kvm_types.h b/arch/powerpc/include/asm/kvm_types.h new file mode 100644 index 000000000000..5d4bffea7d47 --- /dev/null +++ b/arch/powerpc/include/asm/kvm_types.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_PPC_KVM_TYPES_H +#define _ASM_PPC_KVM_TYPES_H + +#if IS_MODULE(CONFIG_KVM_BOOK3S_64_PR) && IS_MODULE(CONFIG_KVM_BOOK3S_64_HV) +#define KVM_SUB_MODULES kvm-pr,kvm-hv +#elif IS_MODULE(CONFIG_KVM_BOOK3S_64_PR) +#define KVM_SUB_MODULES kvm-pr +#elif IS_MODULE(CONFIG_KVM_BOOK3S_64_HV) +#define KVM_SUB_MODULES kvm-hv +#else +#undef KVM_SUB_MODULES +#endif + +#endif diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 9488efea2dba..8e1dcfded1de 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -509,7 +509,7 @@ bool kvm_irq_has_notifier(struct kvm *kvm, unsigned irqchip, unsigned pin) return false; } -EXPORT_SYMBOL_GPL(kvm_irq_has_notifier); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_irq_has_notifier); void kvm_notify_acked_gsi(struct kvm *kvm, int gsi) { diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3f0216045e42..0f8cf16ae992 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -816,7 +816,7 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, return r; } -EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_get_pfn); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE @@ -912,7 +912,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long return ret && !i ? ret : i; } -EXPORT_SYMBOL_GPL(kvm_gmem_populate); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gmem_populate); #endif static struct kmem_cache *kvm_gmem_inode_cachep; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 18f832bf3a27..dc626b70d8d0 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -79,22 +79,22 @@ MODULE_LICENSE("GPL"); /* Architectures should define their poll value according to the halt latency */ unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT; module_param(halt_poll_ns, uint, 0644); -EXPORT_SYMBOL_GPL(halt_poll_ns); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns); /* Default doubles per-vcpu halt_poll_ns. */ unsigned int halt_poll_ns_grow = 2; module_param(halt_poll_ns_grow, uint, 0644); -EXPORT_SYMBOL_GPL(halt_poll_ns_grow); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_grow); /* The start value to grow halt_poll_ns from */ unsigned int halt_poll_ns_grow_start = 10000; /* 10us */ module_param(halt_poll_ns_grow_start, uint, 0644); -EXPORT_SYMBOL_GPL(halt_poll_ns_grow_start); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_grow_start); /* Default resets per-vcpu halt_poll_ns . */ unsigned int halt_poll_ns_shrink; module_param(halt_poll_ns_shrink, uint, 0644); -EXPORT_SYMBOL_GPL(halt_poll_ns_shrink); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(halt_poll_ns_shrink); /* * Allow direct access (from KVM or the CPU) without MMU notifier protection @@ -118,7 +118,7 @@ static __read_mostly struct preempt_ops kvm_preempt_ops; static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); struct dentry *kvm_debugfs_dir; -EXPORT_SYMBOL_GPL(kvm_debugfs_dir); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_debugfs_dir); static const struct file_operations stat_fops_per_vm; @@ -226,7 +226,7 @@ void vcpu_load(struct kvm_vcpu *vcpu) kvm_arch_vcpu_load(vcpu, cpu); put_cpu(); } -EXPORT_SYMBOL_GPL(vcpu_load); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(vcpu_load); void vcpu_put(struct kvm_vcpu *vcpu) { @@ -236,7 +236,7 @@ void vcpu_put(struct kvm_vcpu *vcpu) __this_cpu_write(kvm_running_vcpu, NULL); preempt_enable(); } -EXPORT_SYMBOL_GPL(vcpu_put); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(vcpu_put); /* TODO: merge with kvm_arch_vcpu_should_kick */ static bool kvm_request_needs_ipi(struct kvm_vcpu *vcpu, unsigned req) @@ -353,7 +353,7 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req) { return kvm_make_all_cpus_request_except(kvm, req, NULL); } -EXPORT_SYMBOL_GPL(kvm_make_all_cpus_request); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_make_all_cpus_request); void kvm_flush_remote_tlbs(struct kvm *kvm) { @@ -374,7 +374,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) ++kvm->stat.generic.remote_tlb_flush; } -EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_flush_remote_tlbs); void kvm_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages) { @@ -563,7 +563,7 @@ void kvm_destroy_vcpus(struct kvm *kvm) atomic_set(&kvm->online_vcpus, 0); } -EXPORT_SYMBOL_GPL(kvm_destroy_vcpus); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_destroy_vcpus); #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) @@ -1488,7 +1488,7 @@ void kvm_put_kvm_no_destroy(struct kvm *kvm) { WARN_ON(refcount_dec_and_test(&kvm->users_count)); } -EXPORT_SYMBOL_GPL(kvm_put_kvm_no_destroy); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_put_kvm_no_destroy); static int kvm_vm_release(struct inode *inode, struct file *filp) { @@ -1520,7 +1520,7 @@ int kvm_trylock_all_vcpus(struct kvm *kvm) } return -EINTR; } -EXPORT_SYMBOL_GPL(kvm_trylock_all_vcpus); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_trylock_all_vcpus); int kvm_lock_all_vcpus(struct kvm *kvm) { @@ -1545,7 +1545,7 @@ int kvm_lock_all_vcpus(struct kvm *kvm) } return r; } -EXPORT_SYMBOL_GPL(kvm_lock_all_vcpus); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lock_all_vcpus); void kvm_unlock_all_vcpus(struct kvm *kvm) { @@ -1557,7 +1557,7 @@ void kvm_unlock_all_vcpus(struct kvm *kvm) kvm_for_each_vcpu(i, vcpu, kvm) mutex_unlock(&vcpu->mutex); } -EXPORT_SYMBOL_GPL(kvm_unlock_all_vcpus); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_unlock_all_vcpus); /* * Allocation size is twice as large as the actual dirty bitmap size. @@ -2266,7 +2266,7 @@ int __kvm_set_memory_region(struct kvm *kvm, kfree(new); return r; } -EXPORT_SYMBOL_GPL(__kvm_set_memory_region); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_set_memory_region); int kvm_set_memory_region(struct kvm *kvm, const struct kvm_userspace_memory_region2 *mem) @@ -2278,7 +2278,7 @@ int kvm_set_memory_region(struct kvm *kvm, mutex_unlock(&kvm->slots_lock); return r; } -EXPORT_SYMBOL_GPL(kvm_set_memory_region); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_memory_region); static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm, struct kvm_userspace_memory_region2 *mem) @@ -2336,7 +2336,7 @@ int kvm_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log, *is_dirty = 1; return 0; } -EXPORT_SYMBOL_GPL(kvm_get_dirty_log); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dirty_log); #else /* CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT */ /** @@ -2771,7 +2771,7 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn) { return __gfn_to_memslot(kvm_memslots(kvm), gfn); } -EXPORT_SYMBOL_GPL(gfn_to_memslot); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gfn_to_memslot); struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn) { @@ -2805,7 +2805,7 @@ struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn return NULL; } -EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_gfn_to_memslot); bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) { @@ -2813,7 +2813,7 @@ bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) return kvm_is_visible_memslot(memslot); } -EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_is_visible_gfn); bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) { @@ -2821,7 +2821,7 @@ bool kvm_vcpu_is_visible_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) return kvm_is_visible_memslot(memslot); } -EXPORT_SYMBOL_GPL(kvm_vcpu_is_visible_gfn); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_is_visible_gfn); unsigned long kvm_host_page_size(struct kvm_vcpu *vcpu, gfn_t gfn) { @@ -2878,19 +2878,19 @@ unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot, { return gfn_to_hva_many(slot, gfn, NULL); } -EXPORT_SYMBOL_GPL(gfn_to_hva_memslot); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gfn_to_hva_memslot); unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn) { return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL); } -EXPORT_SYMBOL_GPL(gfn_to_hva); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gfn_to_hva); unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn) { return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL); } -EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_gfn_to_hva); /* * Return the hva of a @gfn and the R/W attribute if possible. @@ -2954,7 +2954,7 @@ void kvm_release_page_clean(struct page *page) kvm_set_page_accessed(page); put_page(page); } -EXPORT_SYMBOL_GPL(kvm_release_page_clean); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_release_page_clean); void kvm_release_page_dirty(struct page *page) { @@ -2964,7 +2964,7 @@ void kvm_release_page_dirty(struct page *page) kvm_set_page_dirty(page); kvm_release_page_clean(page); } -EXPORT_SYMBOL_GPL(kvm_release_page_dirty); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_release_page_dirty); static kvm_pfn_t kvm_resolve_pfn(struct kvm_follow_pfn *kfp, struct page *page, struct follow_pfnmap_args *map, bool writable) @@ -3293,7 +3293,7 @@ kvm_pfn_t __kvm_faultin_pfn(const struct kvm_memory_slot *slot, gfn_t gfn, return kvm_follow_pfn(&kfp); } -EXPORT_SYMBOL_GPL(__kvm_faultin_pfn); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_faultin_pfn); int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn, struct page **pages, int nr_pages) @@ -3310,7 +3310,7 @@ int kvm_prefetch_pages(struct kvm_memory_slot *slot, gfn_t gfn, return get_user_pages_fast_only(addr, nr_pages, FOLL_WRITE, pages); } -EXPORT_SYMBOL_GPL(kvm_prefetch_pages); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_prefetch_pages); /* * Do not use this helper unless you are absolutely certain the gfn _must_ be @@ -3329,7 +3329,7 @@ struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn) return kvm_pfn_to_refcounted_page(pfn); } -EXPORT_SYMBOL_GPL(gfn_to_page); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(gfn_to_page); int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, bool writable) @@ -3363,7 +3363,7 @@ int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, return map->hva ? 0 : -EFAULT; } -EXPORT_SYMBOL_GPL(__kvm_vcpu_map); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_vcpu_map); void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map) { @@ -3391,7 +3391,7 @@ void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map) map->page = NULL; map->pinned_page = NULL; } -EXPORT_SYMBOL_GPL(kvm_vcpu_unmap); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_unmap); void kvm_release_pfn_clean(kvm_pfn_t pfn) { @@ -3478,7 +3478,7 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, return __kvm_read_guest_page(slot, gfn, data, offset, len); } -EXPORT_SYMBOL_GPL(kvm_read_guest_page); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest_page); int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, int len) @@ -3487,7 +3487,7 @@ int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, return __kvm_read_guest_page(slot, gfn, data, offset, len); } -EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_read_guest_page); int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) { @@ -3507,7 +3507,7 @@ int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len) } return 0; } -EXPORT_SYMBOL_GPL(kvm_read_guest); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest); int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len) { @@ -3527,7 +3527,7 @@ int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned l } return 0; } -EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_read_guest); static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn, void *data, int offset, unsigned long len) @@ -3555,7 +3555,7 @@ int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa, return __kvm_read_guest_atomic(slot, gfn, data, offset, len); } -EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_read_guest_atomic); static int __kvm_write_guest_page(struct kvm *kvm, struct kvm_memory_slot *memslot, gfn_t gfn, @@ -3581,7 +3581,7 @@ int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, return __kvm_write_guest_page(kvm, slot, gfn, data, offset, len); } -EXPORT_SYMBOL_GPL(kvm_write_guest_page); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest_page); int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, const void *data, int offset, int len) @@ -3590,7 +3590,7 @@ int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, return __kvm_write_guest_page(vcpu->kvm, slot, gfn, data, offset, len); } -EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_write_guest_page); int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, unsigned long len) @@ -3611,7 +3611,7 @@ int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, } return 0; } -EXPORT_SYMBOL_GPL(kvm_write_guest); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest); int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, unsigned long len) @@ -3632,7 +3632,7 @@ int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, } return 0; } -EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_write_guest); static int __kvm_gfn_to_hva_cache_init(struct kvm_memslots *slots, struct gfn_to_hva_cache *ghc, @@ -3681,7 +3681,7 @@ int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc, struct kvm_memslots *slots = kvm_memslots(kvm); return __kvm_gfn_to_hva_cache_init(slots, ghc, gpa, len); } -EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_gfn_to_hva_cache_init); int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned int offset, @@ -3712,14 +3712,14 @@ int kvm_write_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, return 0; } -EXPORT_SYMBOL_GPL(kvm_write_guest_offset_cached); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest_offset_cached); int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned long len) { return kvm_write_guest_offset_cached(kvm, ghc, data, 0, len); } -EXPORT_SYMBOL_GPL(kvm_write_guest_cached); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_write_guest_cached); int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned int offset, @@ -3749,14 +3749,14 @@ int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, return 0; } -EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest_offset_cached); int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned long len) { return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len); } -EXPORT_SYMBOL_GPL(kvm_read_guest_cached); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_read_guest_cached); int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) { @@ -3776,7 +3776,7 @@ int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len) } return 0; } -EXPORT_SYMBOL_GPL(kvm_clear_guest); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_clear_guest); void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *memslot, @@ -3802,7 +3802,7 @@ void mark_page_dirty_in_slot(struct kvm *kvm, set_bit_le(rel_gfn, memslot->dirty_bitmap); } } -EXPORT_SYMBOL_GPL(mark_page_dirty_in_slot); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(mark_page_dirty_in_slot); void mark_page_dirty(struct kvm *kvm, gfn_t gfn) { @@ -3811,7 +3811,7 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn) memslot = gfn_to_memslot(kvm, gfn); mark_page_dirty_in_slot(kvm, memslot, gfn); } -EXPORT_SYMBOL_GPL(mark_page_dirty); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(mark_page_dirty); void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) { @@ -3820,7 +3820,7 @@ void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn) memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); mark_page_dirty_in_slot(vcpu->kvm, memslot, gfn); } -EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_mark_page_dirty); void kvm_sigset_activate(struct kvm_vcpu *vcpu) { @@ -4057,7 +4057,7 @@ void kvm_vcpu_halt(struct kvm_vcpu *vcpu) trace_kvm_vcpu_wakeup(halt_ns, waited, vcpu_valid_wakeup(vcpu)); } -EXPORT_SYMBOL_GPL(kvm_vcpu_halt); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_halt); bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) { @@ -4069,7 +4069,7 @@ bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) return false; } -EXPORT_SYMBOL_GPL(kvm_vcpu_wake_up); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_wake_up); #ifndef CONFIG_S390 /* @@ -4121,7 +4121,7 @@ void __kvm_vcpu_kick(struct kvm_vcpu *vcpu, bool wait) out: put_cpu(); } -EXPORT_SYMBOL_GPL(__kvm_vcpu_kick); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_vcpu_kick); #endif /* !CONFIG_S390 */ int kvm_vcpu_yield_to(struct kvm_vcpu *target) @@ -4142,7 +4142,7 @@ int kvm_vcpu_yield_to(struct kvm_vcpu *target) return ret; } -EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_yield_to); /* * Helper that checks whether a VCPU is eligible for directed yield. @@ -4266,7 +4266,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) /* Ensure vcpu is not eligible during next spinloop */ kvm_vcpu_set_dy_eligible(me, false); } -EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_vcpu_on_spin); static bool kvm_page_in_dirty_ring(struct kvm *kvm, unsigned long pgoff) { @@ -5192,7 +5192,7 @@ bool kvm_are_all_memslots_empty(struct kvm *kvm) return true; } -EXPORT_SYMBOL_GPL(kvm_are_all_memslots_empty); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_are_all_memslots_empty); static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, struct kvm_enable_cap *cap) @@ -5649,7 +5649,7 @@ bool file_is_kvm(struct file *file) { return file && file->f_op == &kvm_vm_fops; } -EXPORT_SYMBOL_GPL(file_is_kvm); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(file_is_kvm); static int kvm_dev_ioctl_create_vm(unsigned long type) { @@ -5749,10 +5749,10 @@ static struct miscdevice kvm_dev = { #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING bool enable_virt_at_load = true; module_param(enable_virt_at_load, bool, 0444); -EXPORT_SYMBOL_GPL(enable_virt_at_load); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_virt_at_load); __visible bool kvm_rebooting; -EXPORT_SYMBOL_GPL(kvm_rebooting); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_rebooting); static DEFINE_PER_CPU(bool, virtualization_enabled); static DEFINE_MUTEX(kvm_usage_lock); @@ -5903,7 +5903,7 @@ int kvm_enable_virtualization(void) --kvm_usage_count; return r; } -EXPORT_SYMBOL_GPL(kvm_enable_virtualization); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_virtualization); void kvm_disable_virtualization(void) { @@ -5916,7 +5916,7 @@ void kvm_disable_virtualization(void) cpuhp_remove_state(CPUHP_AP_KVM_ONLINE); kvm_arch_disable_virtualization(); } -EXPORT_SYMBOL_GPL(kvm_disable_virtualization); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_disable_virtualization); static int kvm_init_virtualization(void) { @@ -6054,7 +6054,7 @@ int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, r = __kvm_io_bus_write(vcpu, bus, &range, val); return r < 0 ? r : 0; } -EXPORT_SYMBOL_GPL(kvm_io_bus_write); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_write); /* kvm_io_bus_write_cookie - called under kvm->slots_lock */ int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, @@ -6125,7 +6125,7 @@ int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr, r = __kvm_io_bus_read(vcpu, bus, &range, val); return r < 0 ? r : 0; } -EXPORT_SYMBOL_GPL(kvm_io_bus_read); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_read); /* Caller must hold slots_lock. */ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr, @@ -6236,7 +6236,7 @@ struct kvm_io_device *kvm_io_bus_get_dev(struct kvm *kvm, enum kvm_bus bus_idx, return bus->range[dev_idx].dev; } -EXPORT_SYMBOL_GPL(kvm_io_bus_get_dev); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_io_bus_get_dev); static int kvm_debugfs_open(struct inode *inode, struct file *file, int (*get)(void *, u64 *), int (*set)(void *, u64), @@ -6693,7 +6693,7 @@ struct kvm_vcpu *kvm_get_running_vcpu(void) return vcpu; } -EXPORT_SYMBOL_GPL(kvm_get_running_vcpu); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_running_vcpu); /** * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus. @@ -6833,7 +6833,7 @@ int kvm_init(unsigned vcpu_size, unsigned vcpu_align, struct module *module) kmem_cache_destroy(kvm_vcpu_cache); return r; } -EXPORT_SYMBOL_GPL(kvm_init); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init); void kvm_exit(void) { @@ -6857,7 +6857,7 @@ void kvm_exit(void) kvm_async_pf_deinit(); kvm_irqfd_exit(); } -EXPORT_SYMBOL_GPL(kvm_exit); +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_exit); struct kvm_vm_worker_thread_context { struct kvm *kvm; -- Gitee From 98626ebc3ba0c1e5ad465a21a3400a601c5e08ba Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:47 -0800 Subject: [PATCH 16/78] KVM: x86: Move kvm_rebooting to x86 ANBZ: #40036 commit 4059172b2a78a71d15d8fcd8d3fd8ea1ba65d25b upstream. Move kvm_rebooting, which is only read by x86, to KVM x86 so that it can be moved again to core x86 code. Add a "shutdown" arch hook to facilate setting the flag in KVM x86, along with a pile of comments to provide more context around what KVM x86 is doing and why. Intel-SIG: commit 4059172b2a78 KVM: x86: Move kvm_rebooting to x86 Reviewed-by: Chao Gao Acked-by: Dave Hansen Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-2-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/x86.c | 22 ++++++++++++++++++++++ arch/x86/kvm/x86.h | 1 + include/linux/kvm_host.h | 8 +++++++- virt/kvm/kvm_main.c | 13 ++++++------- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 4560412865c9..7e8793f281a3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -492,6 +492,9 @@ static void drop_user_return_notifiers(void) kvm_on_user_return(&msrs->urn); } +__visible bool kvm_rebooting; +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_rebooting); + /* * Handle a fault on a hardware virtualization (VMX or SVM) instruction. * @@ -12716,6 +12719,25 @@ int kvm_arch_enable_virtualization_cpu(void) return 0; } +void kvm_arch_shutdown(void) +{ + /* + * Set kvm_rebooting to indicate that KVM has asynchronously disabled + * hardware virtualization, i.e. that errors and/or exceptions on SVM + * and VMX instructions are expected and should be ignored. + */ + kvm_rebooting = true; + + /* + * Ensure kvm_rebooting is visible before IPIs are sent to other CPUs + * to disable virtualization. Effectively pairs with the reception of + * the IPI (kvm_rebooting is read in task/exception context, but only + * _needs_ to be read as %true after the IPI function callback disables + * virtualization). + */ + smp_wmb(); +} + void kvm_arch_disable_virtualization_cpu(void) { kvm_x86_call(disable_virtualization_cpu)(); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index cf90b59f4184..87962c358e6a 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -38,6 +38,7 @@ struct kvm_caps { u64 inapplicable_quirks; }; +extern bool kvm_rebooting; void kvm_spurious_fault(void); #define KVM_NESTED_VMENTER_CONSISTENCY_CHECK(consistency_check) \ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 2c8d50d206ca..9085e3e3b5f9 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1594,6 +1594,13 @@ static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {} #endif #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING +/* + * kvm_arch_shutdown() is invoked immediately prior to forcefully disabling + * hardware virtualization on all CPUs via IPI function calls (in preparation + * for shutdown or reboot), e.g. to allow arch code to prepare for disabling + * virtualization while KVM may be actively running vCPUs. + */ +void kvm_arch_shutdown(void); /* * kvm_arch_{enable,disable}_virtualization() are called on one CPU, under * kvm_usage_lock, immediately after/before 0=>1 and 1=>0 transitions of @@ -2360,7 +2367,6 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING extern bool enable_virt_at_load; -extern bool kvm_rebooting; #endif extern unsigned int halt_poll_ns; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index dc626b70d8d0..ceb2c4c802df 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5751,13 +5751,15 @@ bool enable_virt_at_load = true; module_param(enable_virt_at_load, bool, 0444); EXPORT_SYMBOL_FOR_KVM_INTERNAL(enable_virt_at_load); -__visible bool kvm_rebooting; -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_rebooting); - static DEFINE_PER_CPU(bool, virtualization_enabled); static DEFINE_MUTEX(kvm_usage_lock); static int kvm_usage_count; +__weak void kvm_arch_shutdown(void) +{ + +} + __weak void kvm_arch_enable_virtualization(void) { @@ -5811,10 +5813,8 @@ static int kvm_offline_cpu(unsigned int cpu) static void kvm_shutdown(void) { + kvm_arch_shutdown(); /* - * Disable hardware virtualization and set kvm_rebooting to indicate - * that KVM has asynchronously disabled hardware virtualization, i.e. - * that relevant errors and exceptions aren't entirely unexpected. * Some flavors of hardware virtualization need to be disabled before * transferring control to firmware (to perform shutdown/reboot), e.g. * on x86, virtualization can block INIT interrupts, which are used by @@ -5823,7 +5823,6 @@ static void kvm_shutdown(void) * 100% comprehensive. */ pr_info("kvm: exiting hardware virtualization\n"); - kvm_rebooting = true; on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1); } -- Gitee From 582741f37520ba810b6b9a0929827666afe55b89 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:48 -0800 Subject: [PATCH 17/78] KVM: VMX: Move architectural "vmcs" and "vmcs_hdr" structures to public vmx.h ANBZ: #40036 commit 3c75e6a5da3c0dfcd34e5f9df5390804179f2aeb upstream. Move "struct vmcs" and "struct vmcs_hdr" to asm/vmx.h in anticipation of moving VMXON/VMXOFF to the core kernel (VMXON requires a "root" VMCS with the appropriate revision ID in its header). No functional change intended. Intel-SIG: commit 3c75e6a5da3c KVM: VMX: Move architectural "vmcs" and "vmcs_hdr" structures to public vmx.h Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-3-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/vmx.h | 11 +++++++++++ arch/x86/kvm/vmx/vmcs.h | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 03171c59fd92..acf6059d4811 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -20,6 +20,17 @@ #include #include +struct vmcs_hdr { + u32 revision_id:31; + u32 shadow_vmcs:1; +}; + +struct vmcs { + struct vmcs_hdr hdr; + u32 abort; + char data[]; +}; + #define VMCS_CONTROL_BIT(x) BIT(VMX_FEATURE_##x & 0x1f) /* diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 20c6d901ea9e..d504d75f8b8e 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -13,17 +13,6 @@ #define ROL16(val, n) ((u16)(((u16)(val) << (n)) | ((u16)(val) >> (16 - (n))))) -struct vmcs_hdr { - u32 revision_id:31; - u32 shadow_vmcs:1; -}; - -struct vmcs { - struct vmcs_hdr hdr; - u32 abort; - char data[]; -}; - DECLARE_PER_CPU(struct vmcs *, current_vmcs); /* -- Gitee From 3d05e8a5cc051aeb409e8da18ca242b20c94dfcc Mon Sep 17 00:00:00 2001 From: Jun Miao Date: Mon, 25 May 2026 23:28:02 -0400 Subject: [PATCH 18/78] x86: Restrict KVM-induced symbol exports to KVM modules where obvious/possible ANBZ: #40036 commit 6276c67f2bc4aeaf350a7cf889c33c38b3330ea9 upstream. Extend KVM's export macro framework to provide EXPORT_SYMBOL_FOR_KVM(), and use the helper macro to export symbols for KVM throughout x86 if and only if KVM will build one or more modules, and only for those modules. To avoid unnecessary exports when CONFIG_KVM=m but kvm.ko will not be built (because no vendor modules are selected), let arch code #define EXPORT_SYMBOL_FOR_KVM to suppress/override the exports. Note, the set of symbols to restrict to KVM was generated by manual search and audit; any "misses" are due to human error, not some grand plan. [ backport ] Don`t export EXPORT_SYMBO_FOR_KVM(wbinvd_on_all_cpus), since there are some other dirver used it. ERROR: modpost: module drm uses symbol wbinvd_on_all_cpus from namespace module:kvm,kvm-amd,kvm-intel, but does not import it. ERROR: modpost: module i915 uses symbol wbinvd_on_all_cpus from namespace module:kvm,kvm-amd,kvm-intel, but does not import it. Intel-SIG: commit 6276c67f2bc4 x86: Restrict KVM-induced symbol exports to KVM modules where obvious/possible Signed-off-by: Sean Christopherson Signed-off-by: Dave Hansen Acked-by: Kai Huang Tested-by: Kai Huang Link: https://patch.msgid.link/20251112173944.1380633-5-seanjc%40google.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/entry/entry.S | 4 +- arch/x86/entry/entry_64.S | 3 +- arch/x86/entry/entry_64_fred.S | 3 +- arch/x86/events/amd/core.c | 5 ++- arch/x86/events/core.c | 7 ++-- arch/x86/events/intel/lbr.c | 3 +- arch/x86/events/intel/pt.c | 7 ++-- arch/x86/kernel/apic/apic.c | 3 +- arch/x86/kernel/apic/apic_common.c | 3 +- arch/x86/kernel/cpu/amd.c | 4 +- arch/x86/kernel/cpu/bugs.c | 13 +++--- arch/x86/kernel/cpu/bus_lock.c | 3 +- arch/x86/kernel/cpu/common.c | 7 ++-- arch/x86/kernel/cpu/sgx/main.c | 3 +- arch/x86/kernel/cpu/sgx/virt.c | 5 ++- arch/x86/kernel/e820.c | 3 +- arch/x86/kernel/fpu/core.c | 21 +++++----- arch/x86/kernel/fpu/xstate.c | 7 ++-- arch/x86/kernel/hw_breakpoint.c | 3 +- arch/x86/kernel/irq.c | 3 +- arch/x86/kernel/kvm.c | 5 ++- arch/x86/kernel/nmi.c | 5 +-- arch/x86/kernel/process_64.c | 5 +-- arch/x86/kernel/reboot.c | 5 ++- arch/x86/kernel/tsc.c | 1 + arch/x86/lib/cache-smp.c | 3 +- arch/x86/lib/msr.c | 3 ++ arch/x86/mm/pat/memtype.c | 3 +- arch/x86/mm/tlb.c | 5 ++- arch/x86/virt/vmx/tdx/tdx.c | 65 +++++++++++++++--------------- include/linux/export.h | 2 + include/linux/kvm_types.h | 1 + 32 files changed, 120 insertions(+), 93 deletions(-) diff --git a/arch/x86/entry/entry.S b/arch/x86/entry/entry.S index 4e7ecffee762..31f5abd97749 100644 --- a/arch/x86/entry/entry.S +++ b/arch/x86/entry/entry.S @@ -4,6 +4,7 @@ */ #include +#include #include #include #include @@ -43,8 +44,7 @@ SYM_CODE_START_NOALIGN(x86_verw_sel) .word __KERNEL_DS .align L1_CACHE_BYTES, 0xcc SYM_CODE_END(x86_verw_sel); -/* For KVM */ -EXPORT_SYMBOL_GPL(x86_verw_sel); +EXPORT_SYMBOL_FOR_KVM(x86_verw_sel); .popsection diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 8b42fb851eeb..77114b8a04a3 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -19,6 +19,7 @@ * - idtentry: Define exception entry points. */ #include +#include #include #include #include @@ -1607,5 +1608,5 @@ SYM_FUNC_START(clear_bhb_loop) pop %rbp RET SYM_FUNC_END(clear_bhb_loop) -EXPORT_SYMBOL_GPL(clear_bhb_loop) +EXPORT_SYMBOL_FOR_KVM(clear_bhb_loop) STACK_FRAME_NON_STANDARD(clear_bhb_loop) diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S index 80002582213e..2a432e8c3742 100644 --- a/arch/x86/entry/entry_64_fred.S +++ b/arch/x86/entry/entry_64_fred.S @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -145,5 +146,5 @@ SYM_FUNC_START(asm_fred_entry_from_kvm) RET SYM_FUNC_END(asm_fred_entry_from_kvm) -EXPORT_SYMBOL_GPL(asm_fred_entry_from_kvm); +EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm); #endif diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 1deccf511c0c..1914fa846369 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -1546,7 +1547,7 @@ void amd_pmu_enable_virt(void) /* Reload all events */ amd_pmu_reload_virt(); } -EXPORT_SYMBOL_GPL(amd_pmu_enable_virt); +EXPORT_SYMBOL_FOR_KVM(amd_pmu_enable_virt); void amd_pmu_disable_virt(void) { @@ -1563,4 +1564,4 @@ void amd_pmu_disable_virt(void) /* Reload all events */ amd_pmu_reload_virt(); } -EXPORT_SYMBOL_GPL(amd_pmu_disable_virt); +EXPORT_SYMBOL_FOR_KVM(amd_pmu_disable_virt); diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index e4098c502cc6..d98a0275e78e 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -726,7 +727,7 @@ struct perf_guest_switch_msr *perf_guest_get_msrs(int *nr, void *data) { return static_call(x86_pmu_guest_get_msrs)(nr, data); } -EXPORT_SYMBOL_GPL(perf_guest_get_msrs); +EXPORT_SYMBOL_FOR_KVM(perf_guest_get_msrs); /* * There may be PMI landing after enabled=0. The PMI hitting could be before or @@ -3163,7 +3164,7 @@ void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap) cap->pebs_ept = x86_pmu.pebs_ept; cap->mediated = !!(pmu.capabilities & PERF_PMU_CAP_MEDIATED_VPMU); } -EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability); +EXPORT_SYMBOL_FOR_KVM(perf_get_x86_pmu_capability); u64 perf_get_hw_event_config(int hw_event) { @@ -3174,4 +3175,4 @@ u64 perf_get_hw_event_config(int hw_event) return 0; } -EXPORT_SYMBOL_GPL(perf_get_hw_event_config); +EXPORT_SYMBOL_FOR_KVM(perf_get_hw_event_config); diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 91fa2154323e..c5b754d04817 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include @@ -1695,7 +1696,7 @@ void x86_perf_get_lbr(struct x86_pmu_lbr *lbr) lbr->to = x86_pmu.lbr_to; lbr->info = x86_pmu.lbr_info; } -EXPORT_SYMBOL_GPL(x86_perf_get_lbr); +EXPORT_SYMBOL_FOR_KVM(x86_perf_get_lbr); struct event_constraint vlbr_constraint = __EVENT_CONSTRAINT(INTEL_FIXED_VLBR_EVENT, (1ULL << INTEL_PMC_IDX_FIXED_VLBR), diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 7ee8dc80a359..6a63c0399afe 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -80,13 +81,13 @@ u32 intel_pt_validate_cap(u32 *caps, enum pt_capabilities capability) return (c & cd->mask) >> shift; } -EXPORT_SYMBOL_GPL(intel_pt_validate_cap); +EXPORT_SYMBOL_FOR_KVM(intel_pt_validate_cap); u32 intel_pt_validate_hw_cap(enum pt_capabilities cap) { return intel_pt_validate_cap(pt_pmu.caps, cap); } -EXPORT_SYMBOL_GPL(intel_pt_validate_hw_cap); +EXPORT_SYMBOL_FOR_KVM(intel_pt_validate_hw_cap); static ssize_t pt_cap_show(struct device *cdev, struct device_attribute *attr, @@ -1562,7 +1563,7 @@ void intel_pt_handle_vmx(int on) local_irq_restore(flags); } -EXPORT_SYMBOL_GPL(intel_pt_handle_vmx); +EXPORT_SYMBOL_FOR_KVM(intel_pt_handle_vmx); /* * PMU callbacks diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 8e779d788491..2c85e9c765d1 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -36,6 +36,7 @@ #include #include #include +#include #include @@ -2325,7 +2326,7 @@ u32 x86_msi_msg_get_destid(struct msi_msg *msg, bool extid) dest |= msg->arch_addr_hi.destid_8_31 << 8; return dest; } -EXPORT_SYMBOL_GPL(x86_msi_msg_get_destid); +EXPORT_SYMBOL_FOR_KVM(x86_msi_msg_get_destid); static void __init apic_bsp_up_setup(void) { diff --git a/arch/x86/kernel/apic/apic_common.c b/arch/x86/kernel/apic/apic_common.c index 9ef3be866832..2ed3b5c88c7f 100644 --- a/arch/x86/kernel/apic/apic_common.c +++ b/arch/x86/kernel/apic/apic_common.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: GPL-2.0 */ #include +#include #include #include "local.h" @@ -25,7 +26,7 @@ u32 default_cpu_present_to_apicid(int mps_cpu) else return BAD_APICID; } -EXPORT_SYMBOL_GPL(default_cpu_present_to_apicid); +EXPORT_SYMBOL_FOR_KVM(default_cpu_present_to_apicid); /* * Set up the logical destination ID when the APIC operates in logical diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 83809fb58dd8..5e85e645d53f 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -3,7 +3,7 @@ #include #include #include - +#include #include #include #include @@ -1293,7 +1293,7 @@ unsigned long amd_get_dr_addr_mask(unsigned int dr) return per_cpu(amd_dr_addr_mask[dr], smp_processor_id()); } -EXPORT_SYMBOL_GPL(amd_get_dr_addr_mask); +EXPORT_SYMBOL_FOR_KVM(amd_get_dr_addr_mask); static void zenbleed_check_cpu(void *unused) { diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index c5394ec3416b..5998854a5e2e 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -240,7 +241,7 @@ x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) speculation_ctrl_update(tif); } } -EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl); +EXPORT_SYMBOL_FOR_KVM(x86_virt_spec_ctrl); static void x86_amd_ssb_disable(void) { @@ -774,7 +775,7 @@ bool gds_ucode_mitigated(void) return (gds_mitigation == GDS_MITIGATION_FULL || gds_mitigation == GDS_MITIGATION_FULL_LOCKED); } -EXPORT_SYMBOL_GPL(gds_ucode_mitigated); +EXPORT_SYMBOL_FOR_KVM(gds_ucode_mitigated); void update_gds_msr(void) { @@ -2672,18 +2673,16 @@ void x86_spec_ctrl_setup_ap(void) } bool itlb_multihit_kvm_mitigation; -EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation); +EXPORT_SYMBOL_FOR_KVM(itlb_multihit_kvm_mitigation); #undef pr_fmt #define pr_fmt(fmt) "L1TF: " fmt /* Default mitigation for L1TF-affected CPUs */ enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH; -#if IS_ENABLED(CONFIG_KVM_INTEL) -EXPORT_SYMBOL_GPL(l1tf_mitigation); -#endif +EXPORT_SYMBOL_FOR_KVM(l1tf_mitigation); enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO; -EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation); +EXPORT_SYMBOL_FOR_KVM(l1tf_vmx_mitigation); /* * These CPUs all support 44bits physical address space internally in the diff --git a/arch/x86/kernel/cpu/bus_lock.c b/arch/x86/kernel/cpu/bus_lock.c index 842a357358ab..be2f453a1e8a 100644 --- a/arch/x86/kernel/cpu/bus_lock.c +++ b/arch/x86/kernel/cpu/bus_lock.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -290,7 +291,7 @@ bool handle_guest_split_lock(unsigned long ip) force_sig_fault(SIGBUS, BUS_ADRALN, NULL); return false; } -EXPORT_SYMBOL_GPL(handle_guest_split_lock); +EXPORT_SYMBOL_FOR_KVM(handle_guest_split_lock); void bus_lock_init(void) { diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 974f358c03e4..cc4586aa4041 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -491,14 +492,14 @@ void cr4_update_irqsoff(unsigned long set, unsigned long clear) __write_cr4(newval); } } -EXPORT_SYMBOL(cr4_update_irqsoff); +EXPORT_SYMBOL_FOR_KVM(cr4_update_irqsoff); /* Read the CR4 shadow. */ unsigned long cr4_read_shadow(void) { return this_cpu_read(cpu_tlbstate.cr4); } -EXPORT_SYMBOL_GPL(cr4_read_shadow); +EXPORT_SYMBOL_FOR_KVM(cr4_read_shadow); void cr4_init(void) { @@ -753,7 +754,7 @@ void load_direct_gdt(int cpu) gdt_descr.size = GDT_SIZE - 1; load_gdt(&gdt_descr); } -EXPORT_SYMBOL_GPL(load_direct_gdt); +EXPORT_SYMBOL_FOR_KVM(load_direct_gdt); /* Load a fixmap remapping of the per-cpu GDT */ void load_fixmap_gdt(int cpu) diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index e0cbc9c4bfa1..f4275faa8461 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -911,7 +912,7 @@ int sgx_set_attribute(unsigned long *allowed_attributes, fdput(f); return 0; } -EXPORT_SYMBOL_GPL(sgx_set_attribute); +EXPORT_SYMBOL_FOR_KVM(sgx_set_attribute); static int __init sgx_init(void) { diff --git a/arch/x86/kernel/cpu/sgx/virt.c b/arch/x86/kernel/cpu/sgx/virt.c index 7aaa3652e31d..727f2570c8b9 100644 --- a/arch/x86/kernel/cpu/sgx/virt.c +++ b/arch/x86/kernel/cpu/sgx/virt.c @@ -5,6 +5,7 @@ * Copyright(c) 2021 Intel Corporation. */ +#include #include #include #include @@ -363,7 +364,7 @@ int sgx_virt_ecreate(struct sgx_pageinfo *pageinfo, void __user *secs, WARN_ON_ONCE(ret); return 0; } -EXPORT_SYMBOL_GPL(sgx_virt_ecreate); +EXPORT_SYMBOL_FOR_KVM(sgx_virt_ecreate); static int __sgx_virt_einit(void __user *sigstruct, void __user *token, void __user *secs) @@ -432,4 +433,4 @@ int sgx_virt_einit(void __user *sigstruct, void __user *token, return ret; } -EXPORT_SYMBOL_GPL(sgx_virt_einit); +EXPORT_SYMBOL_FOR_KVM(sgx_virt_einit); diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 284c8805f2a6..68885d185b50 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -95,7 +96,7 @@ bool e820__mapped_raw_any(u64 start, u64 end, enum e820_type type) { return _e820__mapped_any(e820_table_firmware, start, end, type); } -EXPORT_SYMBOL_GPL(e820__mapped_raw_any); +EXPORT_SYMBOL_FOR_KVM(e820__mapped_raw_any); bool e820__mapped_any(u64 start, u64 end, enum e820_type type) { diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 09b46f100961..1086bf08281c 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -254,7 +255,7 @@ bool fpu_alloc_guest_fpstate(struct fpu_guest *gfpu) return true; } -EXPORT_SYMBOL_GPL(fpu_alloc_guest_fpstate); +EXPORT_SYMBOL_FOR_KVM(fpu_alloc_guest_fpstate); void fpu_free_guest_fpstate(struct fpu_guest *gfpu) { @@ -269,7 +270,7 @@ void fpu_free_guest_fpstate(struct fpu_guest *gfpu) gfpu->fpstate = NULL; vfree(fps); } -EXPORT_SYMBOL_GPL(fpu_free_guest_fpstate); +EXPORT_SYMBOL_FOR_KVM(fpu_free_guest_fpstate); /* * fpu_enable_guest_xfd_features - Check xfeatures against guest perm and enable @@ -291,7 +292,7 @@ int fpu_enable_guest_xfd_features(struct fpu_guest *guest_fpu, u64 xfeatures) return __xfd_enable_feature(xfeatures, guest_fpu); } -EXPORT_SYMBOL_GPL(fpu_enable_guest_xfd_features); +EXPORT_SYMBOL_FOR_KVM(fpu_enable_guest_xfd_features); #ifdef CONFIG_X86_64 void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) @@ -321,7 +322,7 @@ void fpu_update_guest_xfd(struct fpu_guest *guest_fpu, u64 xfd) fpregs_unlock(); } -EXPORT_SYMBOL_GPL(fpu_update_guest_xfd); +EXPORT_SYMBOL_FOR_KVM(fpu_update_guest_xfd); /** * fpu_sync_guest_vmexit_xfd_state - Synchronize XFD MSR and software state @@ -345,7 +346,7 @@ void fpu_sync_guest_vmexit_xfd_state(void) __this_cpu_write(xfd_state, fps->xfd); } } -EXPORT_SYMBOL_GPL(fpu_sync_guest_vmexit_xfd_state); +EXPORT_SYMBOL_FOR_KVM(fpu_sync_guest_vmexit_xfd_state); #endif /* CONFIG_X86_64 */ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) @@ -387,7 +388,7 @@ int fpu_swap_kvm_fpstate(struct fpu_guest *guest_fpu, bool enter_guest) fpregs_unlock(); return 0; } -EXPORT_SYMBOL_GPL(fpu_swap_kvm_fpstate); +EXPORT_SYMBOL_FOR_KVM(fpu_swap_kvm_fpstate); void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf, unsigned int size, u64 xfeatures, u32 pkru) @@ -406,7 +407,7 @@ void fpu_copy_guest_fpstate_to_uabi(struct fpu_guest *gfpu, void *buf, ustate->xsave.header.xfeatures = XFEATURE_MASK_FPSSE; } } -EXPORT_SYMBOL_GPL(fpu_copy_guest_fpstate_to_uabi); +EXPORT_SYMBOL_FOR_KVM(fpu_copy_guest_fpstate_to_uabi); int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, u64 xcr0, u32 *vpkru) @@ -443,7 +444,7 @@ int fpu_copy_uabi_to_guest_fpstate(struct fpu_guest *gfpu, const void *buf, return copy_uabi_from_kernel_to_xstate(kstate, ustate, vpkru); } -EXPORT_SYMBOL_GPL(fpu_copy_uabi_to_guest_fpstate); +EXPORT_SYMBOL_FOR_KVM(fpu_copy_uabi_to_guest_fpstate); #endif /* CONFIG_KVM */ void kernel_fpu_begin_mask(unsigned int kfpu_mask) @@ -923,7 +924,7 @@ void switch_fpu_return(void) fpregs_restore_userregs(); } -EXPORT_SYMBOL_GPL(switch_fpu_return); +EXPORT_SYMBOL_FOR_KVM(switch_fpu_return); void fpregs_lock_and_load(void) { @@ -958,7 +959,7 @@ void fpregs_assert_state_consistent(void) WARN_ON_FPU(!fpregs_state_valid(fpu, smp_processor_id())); } -EXPORT_SYMBOL_GPL(fpregs_assert_state_consistent); +EXPORT_SYMBOL_FOR_KVM(fpregs_assert_state_consistent); #endif void fpregs_mark_activate(void) diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 4dc6d1591344..049cc6e1713f 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -1044,7 +1045,7 @@ void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) return __raw_xsave_addr(xsave, xfeature_nr); } -EXPORT_SYMBOL_GPL(get_xsave_addr); +EXPORT_SYMBOL_FOR_KVM(get_xsave_addr); /* * Given an xstate feature nr, calculate where in the xsave buffer the state is. @@ -1468,7 +1469,7 @@ void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature) if (addr) memset(addr, 0, xstate_sizes[xfeature]); } -EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component); +EXPORT_SYMBOL_FOR_KVM(fpstate_clear_xstate_component); #endif #ifdef CONFIG_X86_64 @@ -1800,7 +1801,7 @@ u64 xstate_get_guest_group_perm(void) { return xstate_get_group_perm(true); } -EXPORT_SYMBOL_GPL(xstate_get_guest_group_perm); +EXPORT_SYMBOL_FOR_KVM(xstate_get_guest_group_perm); /** * fpu_xstate_prctl - xstate permission operations diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoint.c index b01644c949b2..f846c15f21ca 100644 --- a/arch/x86/kernel/hw_breakpoint.c +++ b/arch/x86/kernel/hw_breakpoint.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -489,7 +490,7 @@ void hw_breakpoint_restore(void) set_debugreg(DR6_RESERVED, 6); set_debugreg(__this_cpu_read(cpu_dr7), 7); } -EXPORT_SYMBOL_GPL(hw_breakpoint_restore); +EXPORT_SYMBOL_FOR_KVM(hw_breakpoint_restore); /* * Handle debug exception notifications. diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index a3028f5a1009..71b80f7ae303 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -375,7 +376,7 @@ void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)) synchronize_rcu(); } } -EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler); +EXPORT_SYMBOL_FOR_KVM(kvm_set_posted_intr_wakeup_handler); /* * Handler for POSTED_INTERRUPT_VECTOR. diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 74f90d9bd447..62e0ea526a96 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -172,7 +173,7 @@ void kvm_async_pf_task_wait_schedule(u32 token) } finish_swait(&n.wq, &wait); } -EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule); +EXPORT_SYMBOL_FOR_KVM(kvm_async_pf_task_wait_schedule); static void apf_task_wake_one(struct kvm_task_sleep_node *n) { @@ -264,7 +265,7 @@ noinstr u32 kvm_read_and_reset_apf_flags(void) return flags; } -EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); +EXPORT_SYMBOL_FOR_KVM(kvm_read_and_reset_apf_flags); noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index eb15d82327ca..d5e651012fb4 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -613,9 +614,7 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx) { exc_nmi(regs); } -#if IS_MODULE(CONFIG_KVM_INTEL) -EXPORT_SYMBOL_GPL(asm_exc_nmi_kvm_vmx); -#endif +EXPORT_SYMBOL_FOR_KVM(asm_exc_nmi_kvm_vmx); #endif #ifdef CONFIG_NMI_CHECK_CPU diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index ead39141b8e5..2180abe2d758 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -304,9 +305,7 @@ void current_save_fsgs(void) save_fsgs(current); local_irq_restore(flags); } -#if IS_ENABLED(CONFIG_KVM) -EXPORT_SYMBOL_GPL(current_save_fsgs); -#endif +EXPORT_SYMBOL_FOR_KVM(current_save_fsgs); static __always_inline void loadseg(enum which_selector which, unsigned short sel) diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 1f87caacf07e..d168e5670d85 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -2,6 +2,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -540,7 +541,7 @@ void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback) rcu_assign_pointer(cpu_emergency_virt_callback, callback); } -EXPORT_SYMBOL_GPL(cpu_emergency_register_virt_callback); +EXPORT_SYMBOL_FOR_KVM(cpu_emergency_register_virt_callback); void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback) { @@ -550,7 +551,7 @@ void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback) rcu_assign_pointer(cpu_emergency_virt_callback, NULL); synchronize_rcu(); } -EXPORT_SYMBOL_GPL(cpu_emergency_unregister_virt_callback); +EXPORT_SYMBOL_FOR_KVM(cpu_emergency_unregister_virt_callback); /* * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index d4324901a4a9..d4da0be24731 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/x86/lib/cache-smp.c b/arch/x86/lib/cache-smp.c index 7c48ff4ae8d1..342a51abb9c3 100644 --- a/arch/x86/lib/cache-smp.c +++ b/arch/x86/lib/cache-smp.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include static void __wbinvd(void *dummy) { @@ -11,7 +12,7 @@ void wbinvd_on_cpu(int cpu) { smp_call_function_single(cpu, __wbinvd, NULL, 1); } -EXPORT_SYMBOL(wbinvd_on_cpu); +EXPORT_SYMBOL_FOR_KVM(wbinvd_on_cpu); int wbinvd_on_all_cpus(void) { diff --git a/arch/x86/lib/msr.c b/arch/x86/lib/msr.c index 47fd9bd6b91d..5936d9255275 100644 --- a/arch/x86/lib/msr.c +++ b/arch/x86/lib/msr.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include @@ -103,6 +104,7 @@ int msr_set_bit(u32 msr, u8 bit) { return __flip_bit(msr, bit, true); } +EXPORT_SYMBOL_FOR_KVM(msr_set_bit); /** * msr_clear_bit - Clear @bit in a MSR @msr. @@ -118,6 +120,7 @@ int msr_clear_bit(u32 msr, u8 bit) { return __flip_bit(msr, bit, false); } +EXPORT_SYMBOL_FOR_KVM(msr_clear_bit); #ifdef CONFIG_TRACEPOINTS void do_trace_write_msr(unsigned int msr, u64 val, int failed) diff --git a/arch/x86/mm/pat/memtype.c b/arch/x86/mm/pat/memtype.c index c64a0bf0ce99..95202d43cd53 100644 --- a/arch/x86/mm/pat/memtype.c +++ b/arch/x86/mm/pat/memtype.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -706,7 +707,7 @@ bool pat_pfn_immune_to_uc_mtrr(unsigned long pfn) cm == _PAGE_CACHE_MODE_UC_MINUS || cm == _PAGE_CACHE_MODE_WC; } -EXPORT_SYMBOL_GPL(pat_pfn_immune_to_uc_mtrr); +EXPORT_SYMBOL_FOR_KVM(pat_pfn_immune_to_uc_mtrr); /** * memtype_reserve_io - Request a memory type mapping for a region of memory diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 2f1914dd6c2b..69c99eae2300 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -1498,7 +1499,7 @@ unsigned long __get_current_cr3_fast(void) VM_BUG_ON(cr3 != __read_cr3()); return cr3; } -EXPORT_SYMBOL_GPL(__get_current_cr3_fast); +EXPORT_SYMBOL_FOR_KVM(__get_current_cr3_fast); /* * Flush one page in the kernel mapping @@ -1639,7 +1640,7 @@ void __flush_tlb_all(void) flush_tlb_local(); } } -EXPORT_SYMBOL_GPL(__flush_tlb_all); +EXPORT_SYMBOL_FOR_KVM(__flush_tlb_all); void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 2ffdafdcb3a7..27098c134df6 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -181,7 +182,7 @@ int tdx_cpu_enable(void) return 0; } -EXPORT_SYMBOL_GPL(tdx_cpu_enable); +EXPORT_SYMBOL_FOR_KVM(tdx_cpu_enable); /* * Add a memory region as a TDX memory block. The caller must make sure @@ -1214,7 +1215,7 @@ int tdx_enable(void) return ret; } -EXPORT_SYMBOL_GPL(tdx_enable); +EXPORT_SYMBOL_FOR_KVM(tdx_enable); static bool is_pamt_page(unsigned long phys) { @@ -1475,13 +1476,13 @@ const struct tdx_sys_info *tdx_get_sysinfo(void) return p; } -EXPORT_SYMBOL_GPL(tdx_get_sysinfo); +EXPORT_SYMBOL_FOR_KVM(tdx_get_sysinfo); u32 tdx_get_nr_guest_keyids(void) { return tdx_nr_guest_keyids; } -EXPORT_SYMBOL_GPL(tdx_get_nr_guest_keyids); +EXPORT_SYMBOL_FOR_KVM(tdx_get_nr_guest_keyids); int tdx_guest_keyid_alloc(void) { @@ -1489,13 +1490,13 @@ int tdx_guest_keyid_alloc(void) tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, GFP_KERNEL); } -EXPORT_SYMBOL_GPL(tdx_guest_keyid_alloc); +EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_alloc); void tdx_guest_keyid_free(unsigned int keyid) { ida_free(&tdx_guest_keyid_pool, keyid); } -EXPORT_SYMBOL_GPL(tdx_guest_keyid_free); +EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_free); static inline u64 tdx_tdr_pa(struct tdx_td *td) { @@ -1524,7 +1525,7 @@ noinstr __flatten u64 tdh_vp_enter(struct tdx_vp *td, struct tdx_module_args *ar return __seamcall_saved_ret(TDH_VP_ENTER, args); } -EXPORT_SYMBOL_GPL(tdh_vp_enter); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_enter); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page) { @@ -1536,7 +1537,7 @@ u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page) tdx_clflush_page(tdcs_page); return seamcall(TDH_MNG_ADDCX, &args); } -EXPORT_SYMBOL_GPL(tdh_mng_addcx); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_addcx); u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2) { @@ -1556,7 +1557,7 @@ u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page return ret; } -EXPORT_SYMBOL_GPL(tdh_mem_page_add); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_add); u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2) { @@ -1575,7 +1576,7 @@ u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u return ret; } -EXPORT_SYMBOL_GPL(tdh_mem_sept_add); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_sept_add); u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) { @@ -1587,7 +1588,7 @@ u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) tdx_clflush_page(tdcx_page); return seamcall(TDH_VP_ADDCX, &args); } -EXPORT_SYMBOL_GPL(tdh_vp_addcx); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx); u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2) { @@ -1606,7 +1607,7 @@ u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u return ret; } -EXPORT_SYMBOL_GPL(tdh_mem_page_aug); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_aug); u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2) { @@ -1623,7 +1624,7 @@ u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u6 return ret; } -EXPORT_SYMBOL_GPL(tdh_mem_range_block); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_range_block); u64 tdh_mng_key_config(struct tdx_td *td) { @@ -1633,7 +1634,7 @@ u64 tdh_mng_key_config(struct tdx_td *td) return seamcall(TDH_MNG_KEY_CONFIG, &args); } -EXPORT_SYMBOL_GPL(tdh_mng_key_config); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_key_config); u64 tdh_mng_create(struct tdx_td *td, u16 hkid) { @@ -1645,7 +1646,7 @@ u64 tdh_mng_create(struct tdx_td *td, u16 hkid) tdx_clflush_page(td->tdr_page); return seamcall(TDH_MNG_CREATE, &args); } -EXPORT_SYMBOL_GPL(tdh_mng_create); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_create); u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp) { @@ -1657,7 +1658,7 @@ u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp) tdx_clflush_page(vp->tdvpr_page); return seamcall(TDH_VP_CREATE, &args); } -EXPORT_SYMBOL_GPL(tdh_vp_create); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_create); u64 tdh_mng_rd(struct tdx_td *td, u64 field, u64 *data) { @@ -1674,7 +1675,7 @@ u64 tdh_mng_rd(struct tdx_td *td, u64 field, u64 *data) return ret; } -EXPORT_SYMBOL_GPL(tdh_mng_rd); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_rd); u64 tdh_mr_extend(struct tdx_td *td, u64 gpa, u64 *ext_err1, u64 *ext_err2) { @@ -1691,7 +1692,7 @@ u64 tdh_mr_extend(struct tdx_td *td, u64 gpa, u64 *ext_err1, u64 *ext_err2) return ret; } -EXPORT_SYMBOL_GPL(tdh_mr_extend); +EXPORT_SYMBOL_FOR_KVM(tdh_mr_extend); u64 tdh_mr_finalize(struct tdx_td *td) { @@ -1701,7 +1702,7 @@ u64 tdh_mr_finalize(struct tdx_td *td) return seamcall(TDH_MR_FINALIZE, &args); } -EXPORT_SYMBOL_GPL(tdh_mr_finalize); +EXPORT_SYMBOL_FOR_KVM(tdh_mr_finalize); u64 tdh_vp_flush(struct tdx_vp *vp) { @@ -1711,7 +1712,7 @@ u64 tdh_vp_flush(struct tdx_vp *vp) return seamcall(TDH_VP_FLUSH, &args); } -EXPORT_SYMBOL_GPL(tdh_vp_flush); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_flush); u64 tdh_mng_vpflushdone(struct tdx_td *td) { @@ -1721,7 +1722,7 @@ u64 tdh_mng_vpflushdone(struct tdx_td *td) return seamcall(TDH_MNG_VPFLUSHDONE, &args); } -EXPORT_SYMBOL_GPL(tdh_mng_vpflushdone); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_vpflushdone); u64 tdh_mng_key_freeid(struct tdx_td *td) { @@ -1731,7 +1732,7 @@ u64 tdh_mng_key_freeid(struct tdx_td *td) return seamcall(TDH_MNG_KEY_FREEID, &args); } -EXPORT_SYMBOL_GPL(tdh_mng_key_freeid); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_key_freeid); u64 tdh_mng_init(struct tdx_td *td, u64 td_params, u64 *extended_err) { @@ -1747,7 +1748,7 @@ u64 tdh_mng_init(struct tdx_td *td, u64 td_params, u64 *extended_err) return ret; } -EXPORT_SYMBOL_GPL(tdh_mng_init); +EXPORT_SYMBOL_FOR_KVM(tdh_mng_init); u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data) { @@ -1764,7 +1765,7 @@ u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data) return ret; } -EXPORT_SYMBOL_GPL(tdh_vp_rd); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_rd); u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask) { @@ -1777,7 +1778,7 @@ u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask) return seamcall(TDH_VP_WR, &args); } -EXPORT_SYMBOL_GPL(tdh_vp_wr); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_wr); u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid) { @@ -1790,7 +1791,7 @@ u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid) /* apicid requires version == 1. */ return seamcall(TDH_VP_INIT | (1ULL << TDX_VERSION_SHIFT), &args); } -EXPORT_SYMBOL_GPL(tdh_vp_init); +EXPORT_SYMBOL_FOR_KVM(tdh_vp_init); /* * TDX ABI defines output operands as PT, OWNER and SIZE. These are TDX defined fomats. @@ -1812,7 +1813,7 @@ u64 tdh_phymem_page_reclaim(struct page *page, u64 *tdx_pt, u64 *tdx_owner, u64 return ret; } -EXPORT_SYMBOL_GPL(tdh_phymem_page_reclaim); +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_reclaim); u64 tdh_mem_track(struct tdx_td *td) { @@ -1822,7 +1823,7 @@ u64 tdh_mem_track(struct tdx_td *td) return seamcall(TDH_MEM_TRACK, &args); } -EXPORT_SYMBOL_GPL(tdh_mem_track); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_track); u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2) { @@ -1839,7 +1840,7 @@ u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u6 return ret; } -EXPORT_SYMBOL_GPL(tdh_mem_page_remove); +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_remove); u64 tdh_phymem_cache_wb(bool resume) { @@ -1849,7 +1850,7 @@ u64 tdh_phymem_cache_wb(bool resume) return seamcall(TDH_PHYMEM_CACHE_WB, &args); } -EXPORT_SYMBOL_GPL(tdh_phymem_cache_wb); +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_cache_wb); u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) { @@ -1859,7 +1860,7 @@ u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } -EXPORT_SYMBOL_GPL(tdh_phymem_page_wbinvd_tdr); +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_tdr); u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page) { @@ -1869,4 +1870,4 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page) return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } -EXPORT_SYMBOL_GPL(tdh_phymem_page_wbinvd_hkid); +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid); diff --git a/include/linux/export.h b/include/linux/export.h index bcda631a3b0c..329a3ef745e3 100644 --- a/include/linux/export.h +++ b/include/linux/export.h @@ -96,10 +96,12 @@ extern struct module __this_module; #define EXPORT_SYMBOL_FOR_MODULES(sym, mods) __EXPORT_SYMBOL(sym, "GPL", "module:" mods) +#ifndef EXPORT_SYMBOL_FOR_KVM #ifdef CONFIG_KVM_EXPORT_SYMBOL #define EXPORT_SYMBOL_FOR_KVM(sym) EXPORT_SYMBOL_NS(sym, KVM) #else #define EXPORT_SYMBOL_FOR_KVM(sym) #endif +#endif #endif /* _LINUX_EXPORT_H */ diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index 44b2411077dc..7713f6bf19ea 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -9,6 +9,7 @@ #include #ifdef KVM_SUB_MODULES +#undef EXPORT_SYMBOL_FOR_KVM #define EXPORT_SYMBOL_FOR_KVM_INTERNAL(symbol) \ EXPORT_SYMBOL_FOR_MODULES(symbol, __stringify(KVM_SUB_MODULES)) #undef EXPORT_SYMBOL_FOR_KVM -- Gitee From b213bf875be38aa77d6050d2d23feb8c24bf68e5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:49 -0800 Subject: [PATCH 19/78] KVM: x86: Move "kvm_rebooting" to kernel as "virt_rebooting" ANBZ: #40036 commit a1450a8156c65d9fe6111627094c26359d2e2274 upstream. Move "kvm_rebooting" to the kernel, exported for KVM, as one of many steps towards extracting the innermost VMXON and EFER.SVME management logic out of KVM and into to core x86. For lack of a better name, call the new file "hw.c", to yield "virt hardware" when combined with its parent directory. No functional change intended. Intel-SIG: commit a1450a8156c6 KVM: x86: Move "kvm_rebooting" to kernel as "virt_rebooting" Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-4-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/virt.h | 11 +++++++++++ arch/x86/kvm/svm/svm.c | 3 ++- arch/x86/kvm/svm/vmenter.S | 10 +++++----- arch/x86/kvm/vmx/tdx.c | 3 ++- arch/x86/kvm/vmx/vmenter.S | 2 +- arch/x86/kvm/vmx/vmx.c | 5 +++-- arch/x86/kvm/x86.c | 14 ++++++-------- arch/x86/kvm/x86.h | 1 - arch/x86/virt/Makefile | 3 +++ arch/x86/virt/hw.c | 7 +++++++ 10 files changed, 40 insertions(+), 19 deletions(-) create mode 100644 arch/x86/include/asm/virt.h diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h new file mode 100644 index 000000000000..131b9bf9ef3c --- /dev/null +++ b/arch/x86/include/asm/virt.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_X86_VIRT_H +#define _ASM_X86_VIRT_H + +#include + +#if IS_ENABLED(CONFIG_KVM_X86) +extern bool virt_rebooting; +#endif + +#endif /* _ASM_X86_VIRT_H */ diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0c2eff9d6c1f..57755183ff80 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -527,7 +528,7 @@ static inline void kvm_cpu_svm_disable(void) static void svm_emergency_disable_virtualization_cpu(void) { - kvm_rebooting = true; + virt_rebooting = true; kvm_cpu_svm_disable(); } diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index 81ecb9e1101d..63589e0f29c2 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -269,16 +269,16 @@ SYM_FUNC_START(__svm_vcpu_run) RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY -10: cmpb $0, kvm_rebooting +10: cmpb $0, virt_rebooting jne 2b ud2 -30: cmpb $0, kvm_rebooting +30: cmpb $0, virt_rebooting jne 4b ud2 -50: cmpb $0, kvm_rebooting +50: cmpb $0, virt_rebooting jne 6b ud2 -70: cmpb $0, kvm_rebooting +70: cmpb $0, virt_rebooting jne 8b ud2 @@ -380,7 +380,7 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) RESTORE_GUEST_SPEC_CTRL_BODY RESTORE_HOST_SPEC_CTRL_BODY -3: cmpb $0, kvm_rebooting +3: cmpb $0, virt_rebooting jne 2b ud2 diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 48f621b2992c..855f3e4622cb 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "capabilities.h" #include "mmu.h" #include "x86_ops.h" @@ -2008,7 +2009,7 @@ int tdx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t fastpath) * TDX_SEAMCALL_VMFAILINVALID. */ if (unlikely((vp_enter_ret & TDX_SW_ERROR) == TDX_SW_ERROR)) { - KVM_BUG_ON(!kvm_rebooting, vcpu->kvm); + KVM_BUG_ON(!virt_rebooting, vcpu->kvm); goto unhandled_exit; } diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 9522d46567f8..5de031edd9a7 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -294,7 +294,7 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL) RET .Lfixup: - cmpb $0, kvm_rebooting + cmpb $0, virt_rebooting jne .Lvmfail ud2 .Lvmfail: diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 737dcb2dd7ac..5967c1f01297 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -711,13 +712,13 @@ void vmx_emergency_disable_virtualization_cpu(void) int cpu = raw_smp_processor_id(); struct loaded_vmcs *v; - kvm_rebooting = true; + virt_rebooting = true; /* * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be * set in task context. If this races with VMX is disabled by an NMI, * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to - * kvm_rebooting set. + * virt_rebooting set. */ if (!(__read_cr4() & X86_CR4_VMXE)) return; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7e8793f281a3..e77bbf1cda40 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -84,6 +84,7 @@ #include #include #include +#include #include #include @@ -492,9 +493,6 @@ static void drop_user_return_notifiers(void) kvm_on_user_return(&msrs->urn); } -__visible bool kvm_rebooting; -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_rebooting); - /* * Handle a fault on a hardware virtualization (VMX or SVM) instruction. * @@ -505,7 +503,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_rebooting); noinstr void kvm_spurious_fault(void) { /* Fault while not rebooting. We want the trace. */ - BUG_ON(!kvm_rebooting); + BUG_ON(!virt_rebooting); } EXPORT_SYMBOL_GPL(kvm_spurious_fault); @@ -12722,16 +12720,16 @@ int kvm_arch_enable_virtualization_cpu(void) void kvm_arch_shutdown(void) { /* - * Set kvm_rebooting to indicate that KVM has asynchronously disabled + * Set virt_rebooting to indicate that KVM has asynchronously disabled * hardware virtualization, i.e. that errors and/or exceptions on SVM * and VMX instructions are expected and should be ignored. */ - kvm_rebooting = true; + virt_rebooting = true; /* - * Ensure kvm_rebooting is visible before IPIs are sent to other CPUs + * Ensure virt_rebooting is visible before IPIs are sent to other CPUs * to disable virtualization. Effectively pairs with the reception of - * the IPI (kvm_rebooting is read in task/exception context, but only + * the IPI (virt_rebooting is read in task/exception context, but only * _needs_ to be read as %true after the IPI function callback disables * virtualization). */ diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 87962c358e6a..cf90b59f4184 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -38,7 +38,6 @@ struct kvm_caps { u64 inapplicable_quirks; }; -extern bool kvm_rebooting; void kvm_spurious_fault(void); #define KVM_NESTED_VMENTER_CONSISTENCY_CHECK(consistency_check) \ diff --git a/arch/x86/virt/Makefile b/arch/x86/virt/Makefile index 1e36502cd738..fa378ab08b6b 100644 --- a/arch/x86/virt/Makefile +++ b/arch/x86/virt/Makefile @@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only + obj-y += vmx/ + +obj-$(subst m,y,$(CONFIG_KVM_X86)) += hw.o diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index e69de29bb2d1..df3dc18d19b4 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +#include + +__visible bool virt_rebooting; +EXPORT_SYMBOL_FOR_KVM(virt_rebooting); -- Gitee From de353eaf458807120c670a73e2bbfe643fcbd8f1 Mon Sep 17 00:00:00 2001 From: Hou Wenlong Date: Tue, 13 Jan 2026 19:56:50 +0800 Subject: [PATCH 20/78] KVM: VMX: Don't register posted interrupt wakeup handler if alloc_kvm_area() fails ANBZ: #40036 commit 6c8512a5b7f44caf981cee4ffa2a4ac73e627732 upstream. Unregistering the posted interrupt wakeup handler only happens during hardware unsetup. Therefore, if alloc_kvm_area() fails and continue to register the posted interrupt wakeup handler, this will leave the global posted interrupt wakeup handler pointer in an incorrect state. Although it should not be an issue, it's still better to change it. Intel-SIG: commit 6c8512a5b7f4 KVM: VMX: Don't register posted interrupt wakeup handler if alloc_kvm_area() fails Signed-off-by: Hou Wenlong Fixes: ec5a4919fa7b ("KVM: VMX: Unregister posted interrupt wakeup handler on hardware unsetup") Link: https://patch.msgid.link/0ac6908b608cf80eab7437004334fedd0f5f5317.1768304590.git.houwenlong.hwl@antgroup.com [sean: use a goto] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/vmx.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 5967c1f01297..90b531a8ceb2 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8642,8 +8642,8 @@ __init int vmx_hardware_setup(void) vmx_set_cpu_caps(); r = alloc_kvm_area(); - if (r && nested) - nested_vmx_hardware_unsetup(); + if (r) + goto err_kvm_area; kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); @@ -8668,6 +8668,12 @@ __init int vmx_hardware_setup(void) if (!static_cpu_has(X86_FEATURE_SELFSNOOP)) kvm_caps.supported_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; + + return 0; + + err_kvm_area: + if (nested) + nested_vmx_hardware_unsetup(); return r; } -- Gitee From 8826749e0886222838846bca19c40cf5104b6f78 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:50 -0800 Subject: [PATCH 21/78] KVM: VMX: Unconditionally allocate root VMCSes during boot CPU bringup ANBZ: #40036 commit 405b7c27934eaabbcc52ccfbaeb22ef966b6b5f0 upstream. Allocate the root VMCS (misleading called "vmxarea" and "kvm_area" in KVM) for each possible CPU during early boot CPU bringup, before early TDX initialization, so that TDX can eventually do VMXON on-demand (to make SEAMCALLs) without needing to load kvm-intel.ko. Allocate the pages early on, e.g. instead of trying to do so on-demand, to avoid having to juggle allocation failures at runtime. Opportunistically rename the per-CPU pointers to better reflect the role of the VMCS. Use Intel's "root VMCS" terminology, e.g. from various VMCS patents[1][2] and older SDMs, not the more opaque "VMXON region" used in recent versions of the SDM. While it's possible the VMCS passed to VMXON no longer serves as _the_ root VMCS on modern CPUs, it is still in effect a "root mode VMCS", as described in the patents. Intel-SIG: commit 405b7c27934e KVM: VMX: Unconditionally allocate root VMCSes during boot CPU bringup Link: https://patentimages.storage.googleapis.com/c7/e4/32/d7a7def5580667/WO2013101191A1.pdf [1] Link: https://patentimages.storage.googleapis.com/13/f6/8d/1361fab8c33373/US20080163205A1.pdf [2] Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-5-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/virt.h | 13 ++++++- arch/x86/kernel/cpu/common.c | 2 + arch/x86/kvm/vmx/vmx.c | 58 ++-------------------------- arch/x86/virt/hw.c | 75 +++++++++++++++++++++++++++++++++++- 4 files changed, 91 insertions(+), 57 deletions(-) diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h index 131b9bf9ef3c..0da6db4f5b0c 100644 --- a/arch/x86/include/asm/virt.h +++ b/arch/x86/include/asm/virt.h @@ -2,10 +2,21 @@ #ifndef _ASM_X86_VIRT_H #define _ASM_X86_VIRT_H -#include +#include + +#include #if IS_ENABLED(CONFIG_KVM_X86) extern bool virt_rebooting; + +void __init x86_virt_init(void); + +#if IS_ENABLED(CONFIG_KVM_INTEL) +DECLARE_PER_CPU(struct vmcs *, root_vmcs); +#endif + +#else +static __always_inline void x86_virt_init(void) {} #endif #endif /* _ASM_X86_VIRT_H */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index cc4586aa4041..40bbdfee4f09 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -69,6 +69,7 @@ #include #include #include +#include #include "cpu.h" @@ -2077,6 +2078,7 @@ static __init void identify_boot_cpu(void) cpu_detect_tlb(&boot_cpu_data); setup_cr_pinning(); + x86_virt_init(); tsx_init(); tdx_init(); lkgs_init(); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 90b531a8ceb2..a477da4eda08 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -465,7 +465,6 @@ noinline void invept_error(unsigned long ext, u64 eptp, gpa_t gpa) ext, eptp, gpa); } -static DEFINE_PER_CPU(struct vmcs *, vmxarea); DEFINE_PER_CPU(struct vmcs *, current_vmcs); /* * We maintain a per-CPU linked-list of VMCS loaded on that CPU. This is needed @@ -2777,6 +2776,9 @@ static bool __kvm_is_vmx_supported(void) return false; } + if (!per_cpu(root_vmcs, cpu)) + return false; + return true; } @@ -2836,7 +2838,7 @@ static int kvm_cpu_vmxon(u64 vmxon_pointer) int vmx_enable_virtualization_cpu(void) { int cpu = raw_smp_processor_id(); - u64 phys_addr = __pa(per_cpu(vmxarea, cpu)); + u64 phys_addr = __pa(per_cpu(root_vmcs, cpu)); int r; if (cr4_read_shadow() & X86_CR4_VMXE) @@ -2960,47 +2962,6 @@ int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs) return -ENOMEM; } -static void free_kvm_area(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - free_vmcs(per_cpu(vmxarea, cpu)); - per_cpu(vmxarea, cpu) = NULL; - } -} - -static __init int alloc_kvm_area(void) -{ - int cpu; - - for_each_possible_cpu(cpu) { - struct vmcs *vmcs; - - vmcs = alloc_vmcs_cpu(false, cpu, GFP_KERNEL); - if (!vmcs) { - free_kvm_area(); - return -ENOMEM; - } - - /* - * When eVMCS is enabled, alloc_vmcs_cpu() sets - * vmcs->revision_id to KVM_EVMCS_VERSION instead of - * revision_id reported by MSR_IA32_VMX_BASIC. - * - * However, even though not explicitly documented by - * TLFS, VMXArea passed as VMXON argument should - * still be marked with revision_id reported by - * physical CPU. - */ - if (kvm_is_using_evmcs()) - vmcs->hdr.revision_id = vmcs_config.revision_id; - - per_cpu(vmxarea, cpu) = vmcs; - } - return 0; -} - static void fix_pmode_seg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save) { @@ -8342,8 +8303,6 @@ void vmx_hardware_unsetup(void) if (nested) nested_vmx_hardware_unsetup(); - - free_kvm_area(); } void vmx_vm_destroy(struct kvm *kvm) @@ -8641,10 +8600,6 @@ __init int vmx_hardware_setup(void) vmx_set_cpu_caps(); - r = alloc_kvm_area(); - if (r) - goto err_kvm_area; - kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); /* @@ -8670,11 +8625,6 @@ __init int vmx_hardware_setup(void) kvm_caps.inapplicable_quirks &= ~KVM_X86_QUIRK_IGNORE_GUEST_PAT; return 0; - - err_kvm_area: - if (nested) - nested_vmx_hardware_unsetup(); - return r; } static void vmx_cleanup_l1d_flush(void) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index df3dc18d19b4..1e57248159cb 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -1,7 +1,78 @@ // SPDX-License-Identifier: GPL-2.0-only -#include +#include +#include +#include + #include +#include +#include -#include +#include +#include + #include +#include __visible bool virt_rebooting; EXPORT_SYMBOL_FOR_KVM(virt_rebooting); + +#if IS_ENABLED(CONFIG_KVM_INTEL) +DEFINE_PER_CPU(struct vmcs *, root_vmcs); +EXPORT_PER_CPU_SYMBOL(root_vmcs); + +static __init void x86_vmx_exit(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + free_page((unsigned long)per_cpu(root_vmcs, cpu)); + per_cpu(root_vmcs, cpu) = NULL; + } +} + +static __init int x86_vmx_init(void) +{ + u64 basic_msr; + u32 rev_id; + int cpu; + + if (!cpu_feature_enabled(X86_FEATURE_VMX)) + return -EOPNOTSUPP; + + rdmsrl(MSR_IA32_VMX_BASIC, basic_msr); + + /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ + if (WARN_ON_ONCE(vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE)) + return -EIO; + + /* + * Even if eVMCS is enabled (or will be enabled?), and even though not + * explicitly documented by TLFS, the root VMCS passed to VMXON should + * still be marked with the revision_id reported by the physical CPU. + */ + rev_id = vmx_basic_vmcs_revision_id(basic_msr); + + for_each_possible_cpu(cpu) { + int node = cpu_to_node(cpu); + struct page *page; + struct vmcs *vmcs; + + page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + if (!page) { + x86_vmx_exit(); + return -ENOMEM; + } + + vmcs = page_address(page); + vmcs->hdr.revision_id = rev_id; + per_cpu(root_vmcs, cpu) = vmcs; + } + + return 0; +} +#else +static __init int x86_vmx_init(void) { return -EOPNOTSUPP; } +#endif + +void __init x86_virt_init(void) +{ + x86_vmx_init(); +} -- Gitee From 396c27fe559971e52e362daf84aa204ec561e112 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:51 -0800 Subject: [PATCH 22/78] x86/virt: Force-clear X86_FEATURE_VMX if configuring root VMCS fails ANBZ: #40036 commit 95e4adb24ff6e876f6d2b960cf922d3c969d4dc4 upstream. If allocating and configuring a root VMCS fails, clear X86_FEATURE_VMX in all CPUs so that KVM doesn't need to manually check root_vmcs. As added bonuses, clearing VMX will reflect that VMX is unusable in /proc/cpuinfo, and will avoid a futile auto-probe of kvm-intel.ko. WARN if allocating a root VMCS page fails, e.g. to help users figure out why VMX is broken in the unlikely scenario something goes sideways during boot (and because the allocation should succeed unless there's a kernel bug). Tweak KVM's error message to suggest checking kernel logs if VMX is unsupported (in addition to checking BIOS). Intel-SIG: commit 95e4adb24ff6 x86/virt: Force-clear X86_FEATURE_VMX if configuring root VMCS fails Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-6-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/vmx.c | 7 ++++--- arch/x86/virt/hw.c | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a477da4eda08..a0c544b9bde2 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2770,14 +2770,15 @@ static bool __kvm_is_vmx_supported(void) return false; } - if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL) || - !this_cpu_has(X86_FEATURE_VMX)) { + if (!this_cpu_has(X86_FEATURE_MSR_IA32_FEAT_CTL)) { pr_err("VMX not enabled (by BIOS) in MSR_IA32_FEAT_CTL on CPU %d\n", cpu); return false; } - if (!per_cpu(root_vmcs, cpu)) + if (!this_cpu_has(X86_FEATURE_VMX)) { + pr_err("VMX not fully enabled on CPU %d. Check kernel logs and/or BIOS\n", cpu); return false; + } return true; } diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 1e57248159cb..bacb6c2b123e 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -28,7 +28,7 @@ static __init void x86_vmx_exit(void) } } -static __init int x86_vmx_init(void) +static __init int __x86_vmx_init(void) { u64 basic_msr; u32 rev_id; @@ -56,7 +56,7 @@ static __init int x86_vmx_init(void) struct vmcs *vmcs; page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); - if (!page) { + if (WARN_ON_ONCE(!page)) { x86_vmx_exit(); return -ENOMEM; } @@ -68,6 +68,16 @@ static __init int x86_vmx_init(void) return 0; } + +static __init int x86_vmx_init(void) +{ + int r; + + r = __x86_vmx_init(); + if (r) + setup_clear_cpu_cap(X86_FEATURE_VMX); + return r; +} #else static __init int x86_vmx_init(void) { return -EOPNOTSUPP; } #endif -- Gitee From 2c2e35bfe705f8549c3c10670bcc34aaedcf5f3e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 7 Jun 2024 17:10:03 -0700 Subject: [PATCH 23/78] KVM: VMX: Remove unnecessary INVEPT[GLOBAL] from hardware enable path ANBZ: #40036 commit 23b2c5088d01dc7dfdb68aab76a7757704f09c6e upstream. Remove the completely pointess global INVEPT, i.e. EPT TLB flush, from KVM's VMX enablement path. KVM always does a targeted TLB flush when using a "new" EPT root, in quotes because "new" simply means a root that isn't currently being used by the vCPU. KVM also _deliberately_ runs with stale TLB entries for defunct roots, i.e. doesn't do a TLB flush when vCPUs stop using roots, precisely because KVM does the flush on first use. As called out by the comment in kvm_mmu_load(), the reason KVM flushes on first use is because KVM can't guarantee the correctness of past hypervisors. Jumping back to the global INVEPT, when the painfully terse commit 1439442c7b25 ("KVM: VMX: Enable EPT feature for KVM") was added, the effective TLB flush being performed was: static void vmx_flush_tlb(struct kvm_vcpu *vcpu) { vpid_sync_vcpu_all(to_vmx(vcpu)); } I.e. KVM was not flushing EPT TLB entries when allocating a "new" root, which very strongly suggests that the global INVEPT during hardware enabling was a misguided hack that addressed the most obvious symptom, but failed to fix the underlying bug. Intel-SIG: commit 23b2c5088d01 KVM: VMX: Remove unnecessary INVEPT[GLOBAL] from hardware enable path Reviewed-by: Paolo Bonzini Link: https://lore.kernel.org/r/20240608001003.3296640-1-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/vmx.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a0c544b9bde2..9c950632a11a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2860,9 +2860,6 @@ int vmx_enable_virtualization_cpu(void) return r; } - if (enable_ept) - ept_sync_global(); - return 0; } -- Gitee From 5b332dea015be36299aab9b8ecd36cda633440e1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:52 -0800 Subject: [PATCH 24/78] KVM: VMX: Move core VMXON enablement to kernel ANBZ: #40036 commit 920da4f75519a3fa3fe2fc25458445b561653610 upstream. Move the innermost VMXON+VMXOFF logic out of KVM and into to core x86 so that TDX can (eventually) force VMXON without having to rely on KVM being loaded, e.g. to do SEAMCALLs during initialization. Opportunistically update the comment regarding emergency disabling via NMI to clarify that virt_rebooting will be set by _another_ emergency callback, i.e. that virt_rebooting doesn't need to be set before VMCLEAR, only before _this_ invocation does VMXOFF. Intel-SIG: commit 920da4f75519 KVM: VMX: Move core VMXON enablement to kernel Acked-by: Dave Hansen Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-7-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/events/intel/pt.c | 1 - arch/x86/include/asm/virt.h | 6 +-- arch/x86/kvm/vmx/vmx.c | 73 +++---------------------------- arch/x86/virt/hw.c | 85 ++++++++++++++++++++++++++++++++++++- 4 files changed, 92 insertions(+), 73 deletions(-) diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index 6a63c0399afe..0e7a94f3829f 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -1563,7 +1563,6 @@ void intel_pt_handle_vmx(int on) local_irq_restore(flags); } -EXPORT_SYMBOL_FOR_KVM(intel_pt_handle_vmx); /* * PMU callbacks diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h index 0da6db4f5b0c..cca0210a5c16 100644 --- a/arch/x86/include/asm/virt.h +++ b/arch/x86/include/asm/virt.h @@ -2,8 +2,6 @@ #ifndef _ASM_X86_VIRT_H #define _ASM_X86_VIRT_H -#include - #include #if IS_ENABLED(CONFIG_KVM_X86) @@ -12,7 +10,9 @@ extern bool virt_rebooting; void __init x86_virt_init(void); #if IS_ENABLED(CONFIG_KVM_INTEL) -DECLARE_PER_CPU(struct vmcs *, root_vmcs); +int x86_vmx_enable_virtualization_cpu(void); +int x86_vmx_disable_virtualization_cpu(void); +void x86_vmx_emergency_disable_virtualization_cpu(void); #endif #else diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9c950632a11a..89b3168d322f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -683,41 +683,16 @@ static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx, return ret; } -/* - * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) - * - * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to - * atomically track post-VMXON state, e.g. this may be called in NMI context. - * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. - * faults are guaranteed to be due to the !post-VMXON check unless the CPU is - * magically in RM, VM86, compat mode, or at CPL>0. - */ -static int kvm_cpu_vmxoff(void) -{ - asm goto("1: vmxoff\n\t" - _ASM_EXTABLE(1b, %l[fault]) - ::: "cc", "memory" : fault); - - cr4_clear_bits(X86_CR4_VMXE); - return 0; - -fault: - cr4_clear_bits(X86_CR4_VMXE); - return -EIO; -} - void vmx_emergency_disable_virtualization_cpu(void) { int cpu = raw_smp_processor_id(); struct loaded_vmcs *v; - virt_rebooting = true; - /* * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be - * set in task context. If this races with VMX is disabled by an NMI, - * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to - * virt_rebooting set. + * set in task context. If this races with _another_ emergency call + * from NMI context, VMCLEAR may #UD, but KVM will eat those faults due + * to virt_rebooting being set by the interrupting NMI callback. */ if (!(__read_cr4() & X86_CR4_VMXE)) return; @@ -729,7 +704,7 @@ void vmx_emergency_disable_virtualization_cpu(void) vmcs_clear(v->shadow_vmcs); } - kvm_cpu_vmxoff(); + x86_vmx_emergency_disable_virtualization_cpu(); } static void __loaded_vmcs_clear(void *arg) @@ -2816,34 +2791,9 @@ int vmx_check_processor_compat(void) return 0; } -static int kvm_cpu_vmxon(u64 vmxon_pointer) -{ - u64 msr; - - cr4_set_bits(X86_CR4_VMXE); - - asm goto("1: vmxon %[vmxon_pointer]\n\t" - _ASM_EXTABLE(1b, %l[fault]) - : : [vmxon_pointer] "m"(vmxon_pointer) - : : fault); - return 0; - -fault: - WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", - rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); - cr4_clear_bits(X86_CR4_VMXE); - - return -EFAULT; -} - int vmx_enable_virtualization_cpu(void) { int cpu = raw_smp_processor_id(); - u64 phys_addr = __pa(per_cpu(root_vmcs, cpu)); - int r; - - if (cr4_read_shadow() & X86_CR4_VMXE) - return -EBUSY; /* * This can happen if we hot-added a CPU but failed to allocate @@ -2852,15 +2802,7 @@ int vmx_enable_virtualization_cpu(void) if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) return -EFAULT; - intel_pt_handle_vmx(1); - - r = kvm_cpu_vmxon(phys_addr); - if (r) { - intel_pt_handle_vmx(0); - return r; - } - - return 0; + return x86_vmx_enable_virtualization_cpu(); } static void vmclear_local_loaded_vmcss(void) @@ -2877,12 +2819,9 @@ void vmx_disable_virtualization_cpu(void) { vmclear_local_loaded_vmcss(); - if (kvm_cpu_vmxoff()) - kvm_spurious_fault(); + x86_vmx_disable_virtualization_cpu(); hv_reset_evmcs(); - - intel_pt_handle_vmx(0); } struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index bacb6c2b123e..a298f1489bea 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -15,8 +15,89 @@ __visible bool virt_rebooting; EXPORT_SYMBOL_FOR_KVM(virt_rebooting); #if IS_ENABLED(CONFIG_KVM_INTEL) -DEFINE_PER_CPU(struct vmcs *, root_vmcs); -EXPORT_PER_CPU_SYMBOL(root_vmcs); +static DEFINE_PER_CPU(struct vmcs *, root_vmcs); + +static int x86_virt_cpu_vmxon(void) +{ + u64 vmxon_pointer = __pa(per_cpu(root_vmcs, raw_smp_processor_id())); + u64 msr; + + cr4_set_bits(X86_CR4_VMXE); + + asm goto("1: vmxon %[vmxon_pointer]\n\t" + _ASM_EXTABLE(1b, %l[fault]) + : : [vmxon_pointer] "m"(vmxon_pointer) + : : fault); + return 0; + +fault: + WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", + rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); + cr4_clear_bits(X86_CR4_VMXE); + + return -EFAULT; +} + +int x86_vmx_enable_virtualization_cpu(void) +{ + int r; + + if (cr4_read_shadow() & X86_CR4_VMXE) + return -EBUSY; + + intel_pt_handle_vmx(1); + + r = x86_virt_cpu_vmxon(); + if (r) { + intel_pt_handle_vmx(0); + return r; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM(x86_vmx_enable_virtualization_cpu); + +/* + * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) + * + * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to + * atomically track post-VMXON state, e.g. this may be called in NMI context. + * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. + * faults are guaranteed to be due to the !post-VMXON check unless the CPU is + * magically in RM, VM86, compat mode, or at CPL>0. + */ +int x86_vmx_disable_virtualization_cpu(void) +{ + int r = -EIO; + + asm goto("1: vmxoff\n\t" + _ASM_EXTABLE(1b, %l[fault]) + ::: "cc", "memory" : fault); + r = 0; + +fault: + cr4_clear_bits(X86_CR4_VMXE); + intel_pt_handle_vmx(0); + return r; +} +EXPORT_SYMBOL_FOR_KVM(x86_vmx_disable_virtualization_cpu); + +void x86_vmx_emergency_disable_virtualization_cpu(void) +{ + virt_rebooting = true; + + /* + * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be + * set in task context. If this races with _another_ emergency call + * from NMI context, VMXOFF may #UD, but kernel will eat those faults + * due to virt_rebooting being set by the interrupting NMI callback. + */ + if (!(__read_cr4() & X86_CR4_VMXE)) + return; + + x86_vmx_disable_virtualization_cpu(); +} +EXPORT_SYMBOL_FOR_KVM(x86_vmx_emergency_disable_virtualization_cpu); static __init void x86_vmx_exit(void) { -- Gitee From 1cfba8d50868a8267aac3a0ffb925660acdfae80 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:53 -0800 Subject: [PATCH 25/78] KVM: SVM: Move core EFER.SVME enablement to kernel ANBZ: #40036 commit 32d76cdfa1222c88262da5b12e0b2bba444c96fa upstream. Move the innermost EFER.SVME logic out of KVM and into to core x86 to land the SVM support alongside VMX support. This will allow providing a more unified API from the kernel to KVM, and will allow moving the bulk of the emergency disabling insanity out of KVM without having a weird split between kernel and KVM for SVM vs. VMX. No functional change intended. [backport] rdmsrq->rdmsrl wrmsrq->wrmsrl Intel-SIG: commit 32d76cdfa12 KVM: SVM: Move core EFER.SVME enablement to kernel Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-8-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/virt.h | 6 +++++ arch/x86/kvm/svm/svm.c | 34 +++++------------------- arch/x86/virt/hw.c | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h index cca0210a5c16..9a0753eaa20c 100644 --- a/arch/x86/include/asm/virt.h +++ b/arch/x86/include/asm/virt.h @@ -15,6 +15,12 @@ int x86_vmx_disable_virtualization_cpu(void); void x86_vmx_emergency_disable_virtualization_cpu(void); #endif +#if IS_ENABLED(CONFIG_KVM_AMD) +int x86_svm_enable_virtualization_cpu(void); +int x86_svm_disable_virtualization_cpu(void); +void x86_svm_emergency_disable_virtualization_cpu(void); +#endif + #else static __always_inline void x86_virt_init(void) {} #endif diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 57755183ff80..d1c2a7cbdbb4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -510,27 +510,9 @@ static void __svm_write_tsc_multiplier(u64 multiplier) __this_cpu_write(current_tsc_ratio, multiplier); } -static inline void kvm_cpu_svm_disable(void) -{ - uint64_t efer; - - wrmsrl(MSR_VM_HSAVE_PA, 0); - rdmsrl(MSR_EFER, efer); - if (efer & EFER_SVME) { - /* - * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and - * NMI aren't blocked. - */ - stgi(); - wrmsrl(MSR_EFER, efer & ~EFER_SVME); - } -} - static void svm_emergency_disable_virtualization_cpu(void) { - virt_rebooting = true; - - kvm_cpu_svm_disable(); + wrmsrl(MSR_VM_HSAVE_PA, 0); } static void svm_disable_virtualization_cpu(void) @@ -539,7 +521,8 @@ static void svm_disable_virtualization_cpu(void) if (tsc_scaling) __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); - kvm_cpu_svm_disable(); + x86_svm_disable_virtualization_cpu(); + wrmsrl(MSR_VM_HSAVE_PA, 0); amd_pmu_disable_virt(); } @@ -548,12 +531,12 @@ static int svm_enable_virtualization_cpu(void) { struct svm_cpu_data *sd; - uint64_t efer; int me = raw_smp_processor_id(); + int r; - rdmsrl(MSR_EFER, efer); - if (efer & EFER_SVME) - return -EBUSY; + r = x86_svm_enable_virtualization_cpu(); + if (r) + return r; sd = per_cpu_ptr(&svm_data, me); sd->asid_generation = 1; @@ -561,8 +544,6 @@ static int svm_enable_virtualization_cpu(void) sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; - wrmsrl(MSR_EFER, efer | EFER_SVME); - wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { @@ -573,7 +554,6 @@ static int svm_enable_virtualization_cpu(void) __svm_write_tsc_multiplier(SVM_TSC_RATIO_DEFAULT); } - /* * Get OSVW bits. * diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index a298f1489bea..a088b8eb3f14 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -163,6 +163,59 @@ static __init int x86_vmx_init(void) static __init int x86_vmx_init(void) { return -EOPNOTSUPP; } #endif +#if IS_ENABLED(CONFIG_KVM_AMD) +int x86_svm_enable_virtualization_cpu(void) +{ + u64 efer; + + if (!cpu_feature_enabled(X86_FEATURE_SVM)) + return -EOPNOTSUPP; + + rdmsrl(MSR_EFER, efer); + if (efer & EFER_SVME) + return -EBUSY; + + wrmsrl(MSR_EFER, efer | EFER_SVME); + return 0; +} +EXPORT_SYMBOL_FOR_KVM(x86_svm_enable_virtualization_cpu); + +int x86_svm_disable_virtualization_cpu(void) +{ + int r = -EIO; + u64 efer; + + /* + * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and + * NMI aren't blocked. + */ + asm goto("1: stgi\n\t" + _ASM_EXTABLE(1b, %l[fault]) + ::: "memory" : fault); + r = 0; + +fault: + rdmsrl(MSR_EFER, efer); + wrmsrl(MSR_EFER, efer & ~EFER_SVME); + return r; +} +EXPORT_SYMBOL_FOR_KVM(x86_svm_disable_virtualization_cpu); + +void x86_svm_emergency_disable_virtualization_cpu(void) +{ + u64 efer; + + virt_rebooting = true; + + rdmsrl(MSR_EFER, efer); + if (!(efer & EFER_SVME)) + return; + + x86_svm_disable_virtualization_cpu(); +} +EXPORT_SYMBOL_FOR_KVM(x86_svm_emergency_disable_virtualization_cpu); +#endif + void __init x86_virt_init(void) { x86_vmx_init(); -- Gitee From 77fd114a5336fe171f0c303e866c32ae8c79aa1e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:54 -0800 Subject: [PATCH 26/78] KVM: x86: Move bulk of emergency virtualizaton logic to virt subsystem ANBZ: #40036 commit 428afac5a8ea9c55bb8408e02dc92b8f85bf5f30 upstream. Move the majority of the code related to disabling hardware virtualization in emergency from KVM into the virt subsystem so that virt can take full ownership of the state of SVM/VMX. This will allow refcounting usage of SVM/VMX so that KVM and the TDX subsystem can enable VMX without stomping on each other. To route the emergency callback to the "right" vendor code, add to avoid mixing vendor and generic code, implement a x86_virt_ops structure to track the emergency callback, along with the SVM vs. VMX (vs. "none") feature that is active. To avoid having to choose between SVM and VMX, simply refuse to enable either if both are somehow supported. No known CPU supports both SVM and VMX, and it's comically unlikely such a CPU will ever exist. Leave KVM's clearing of loaded VMCSes and MSR_VM_HSAVE_PA in KVM, via a callback explicitly scoped to KVM. Loading VMCSes and saving/restoring host state are firmly tied to running VMs, and thus are (a) KVM's responsibility and (b) operations that are still exclusively reserved for KVM (as far as in-tree code is concerned). I.e. the contract being established is that non-KVM subsystems can utilize virtualization, but for all intents and purposes cannot act as full-blown hypervisors. Intel-SIG: commit 428afac5a8ea KVM: x86: Move bulk of emergency virtualizaton logic to virt subsystem Reviewed-by: Chao Gao Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-9-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/include/asm/reboot.h | 11 --- arch/x86/include/asm/virt.h | 9 ++- arch/x86/kernel/crash.c | 3 +- arch/x86/kernel/reboot.c | 62 ++-------------- arch/x86/kernel/smp.c | 5 +- arch/x86/kvm/vmx/vmx.c | 11 --- arch/x86/kvm/x86.c | 4 +- arch/x86/virt/hw.c | 124 +++++++++++++++++++++++++++++--- 9 files changed, 136 insertions(+), 95 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 5e78f95b93f3..988e5eb7d25f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -37,7 +37,7 @@ #include #include #include -#include +#include #define __KVM_HAVE_ARCH_VCPU_DEBUGFS diff --git a/arch/x86/include/asm/reboot.h b/arch/x86/include/asm/reboot.h index ecd58ea9a837..a671a1145906 100644 --- a/arch/x86/include/asm/reboot.h +++ b/arch/x86/include/asm/reboot.h @@ -25,17 +25,6 @@ void __noreturn machine_real_restart(unsigned int type); #define MRR_BIOS 0 #define MRR_APM 1 -typedef void (cpu_emergency_virt_cb)(void); -#if IS_ENABLED(CONFIG_KVM_X86) -void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback); -void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback); -void cpu_emergency_disable_virtualization(void); -#else -static inline void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback) {} -static inline void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback) {} -static inline void cpu_emergency_disable_virtualization(void) {} -#endif /* CONFIG_KVM_X86 */ - typedef void (*nmi_shootdown_cb)(int, struct pt_regs*); void nmi_shootdown_cpus(nmi_shootdown_cb callback); void run_crash_ipi_callback(struct pt_regs *regs); diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h index 9a0753eaa20c..2c35534437e0 100644 --- a/arch/x86/include/asm/virt.h +++ b/arch/x86/include/asm/virt.h @@ -4,6 +4,8 @@ #include +typedef void (cpu_emergency_virt_cb)(void); + #if IS_ENABLED(CONFIG_KVM_X86) extern bool virt_rebooting; @@ -12,17 +14,20 @@ void __init x86_virt_init(void); #if IS_ENABLED(CONFIG_KVM_INTEL) int x86_vmx_enable_virtualization_cpu(void); int x86_vmx_disable_virtualization_cpu(void); -void x86_vmx_emergency_disable_virtualization_cpu(void); #endif #if IS_ENABLED(CONFIG_KVM_AMD) int x86_svm_enable_virtualization_cpu(void); int x86_svm_disable_virtualization_cpu(void); -void x86_svm_emergency_disable_virtualization_cpu(void); #endif +int x86_virt_emergency_disable_virtualization_cpu(void); + +void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback); +void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback); #else static __always_inline void x86_virt_init(void) {} +static inline int x86_virt_emergency_disable_virtualization_cpu(void) { return -ENOENT; } #endif #endif /* _ASM_X86_VIRT_H */ diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c index 64ae3a1e5c30..bcd94ef88ac7 100644 --- a/arch/x86/kernel/crash.c +++ b/arch/x86/kernel/crash.c @@ -41,6 +41,7 @@ #include #include #include +#include /* Used while preparing memory map entries for second kernel */ struct crash_memmap_data { @@ -110,7 +111,7 @@ void native_machine_crash_shutdown(struct pt_regs *regs) crash_smp_send_stop(); - cpu_emergency_disable_virtualization(); + x86_virt_emergency_disable_virtualization_cpu(); /* * Disable Intel PT to stop its logging diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index d168e5670d85..374d08efc25d 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -531,50 +532,6 @@ static inline void kb_wait(void) static inline void nmi_shootdown_cpus_on_restart(void); #if IS_ENABLED(CONFIG_KVM_X86) -/* RCU-protected callback to disable virtualization prior to reboot. */ -static cpu_emergency_virt_cb __rcu *cpu_emergency_virt_callback; - -void cpu_emergency_register_virt_callback(cpu_emergency_virt_cb *callback) -{ - if (WARN_ON_ONCE(rcu_access_pointer(cpu_emergency_virt_callback))) - return; - - rcu_assign_pointer(cpu_emergency_virt_callback, callback); -} -EXPORT_SYMBOL_FOR_KVM(cpu_emergency_register_virt_callback); - -void cpu_emergency_unregister_virt_callback(cpu_emergency_virt_cb *callback) -{ - if (WARN_ON_ONCE(rcu_access_pointer(cpu_emergency_virt_callback) != callback)) - return; - - rcu_assign_pointer(cpu_emergency_virt_callback, NULL); - synchronize_rcu(); -} -EXPORT_SYMBOL_FOR_KVM(cpu_emergency_unregister_virt_callback); - -/* - * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during - * reboot. VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if - * GIF=0, i.e. if the crash occurred between CLGI and STGI. - */ -void cpu_emergency_disable_virtualization(void) -{ - cpu_emergency_virt_cb *callback; - - /* - * IRQs must be disabled as KVM enables virtualization in hardware via - * function call IPIs, i.e. IRQs need to be disabled to guarantee - * virtualization stays disabled. - */ - lockdep_assert_irqs_disabled(); - - rcu_read_lock(); - callback = rcu_dereference(cpu_emergency_virt_callback); - if (callback) - callback(); - rcu_read_unlock(); -} static void emergency_reboot_disable_virtualization(void) { @@ -587,16 +544,11 @@ static void emergency_reboot_disable_virtualization(void) * We can't take any locks and we may be on an inconsistent state, so * use NMIs as IPIs to tell the other CPUs to disable VMX/SVM and halt. * - * Do the NMI shootdown even if virtualization is off on _this_ CPU, as - * other CPUs may have virtualization enabled. + * Safely force _this_ CPU out of VMX/SVM operation, and if necessary, + * blast NMIs to force other CPUs out of VMX/SVM as well.k */ - if (rcu_access_pointer(cpu_emergency_virt_callback)) { - /* Safely force _this_ CPU out of VMX/SVM operation. */ - cpu_emergency_disable_virtualization(); - - /* Disable VMX/SVM and halt on other CPUs. */ + if (!x86_virt_emergency_disable_virtualization_cpu()) nmi_shootdown_cpus_on_restart(); - } } #else static void emergency_reboot_disable_virtualization(void) { } @@ -863,10 +815,10 @@ static int crash_nmi_callback(unsigned int val, struct pt_regs *regs) shootdown_callback(cpu, regs); /* - * Prepare the CPU for reboot _after_ invoking the callback so that the - * callback can safely use virtualization instructions, e.g. VMCLEAR. + * Disable virtualization, as both VMX and SVM can block INIT and thus + * prevent AP bringup, e.g. in a kdump kernel or in firmware. */ - cpu_emergency_disable_virtualization(); + x86_virt_emergency_disable_virtualization_cpu(); atomic_dec(&waiting_for_crash_ipi); /* Assume hlt works */ diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index 1e8e6efe8ddf..f43da36a7d05 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -35,6 +35,7 @@ #include #include #include +#include /* * Some notes on x86 processor bugs affecting SMP operation: @@ -124,7 +125,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs) if (raw_smp_processor_id() == atomic_read(&stopping_cpu)) return NMI_HANDLED; - cpu_emergency_disable_virtualization(); + x86_virt_emergency_disable_virtualization_cpu(); stop_this_cpu(NULL); return NMI_HANDLED; @@ -136,7 +137,7 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs) DEFINE_IDTENTRY_SYSVEC(sysvec_reboot) { apic_eoi(); - cpu_emergency_disable_virtualization(); + x86_virt_emergency_disable_virtualization_cpu(); stop_this_cpu(NULL); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 89b3168d322f..ace9eee202aa 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -688,23 +688,12 @@ void vmx_emergency_disable_virtualization_cpu(void) int cpu = raw_smp_processor_id(); struct loaded_vmcs *v; - /* - * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be - * set in task context. If this races with _another_ emergency call - * from NMI context, VMCLEAR may #UD, but KVM will eat those faults due - * to virt_rebooting being set by the interrupting NMI callback. - */ - if (!(__read_cr4() & X86_CR4_VMXE)) - return; - list_for_each_entry(v, &per_cpu(loaded_vmcss_on_cpu, cpu), loaded_vmcss_on_cpu_link) { vmcs_clear(v->vmcs); if (v->shadow_vmcs) vmcs_clear(v->shadow_vmcs); } - - x86_vmx_emergency_disable_virtualization_cpu(); } static void __loaded_vmcs_clear(void *arg) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e77bbf1cda40..12a05ca39513 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -12613,12 +12613,12 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_deliver_sipi_vector); void kvm_arch_enable_virtualization(void) { - cpu_emergency_register_virt_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); + x86_virt_register_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); } void kvm_arch_disable_virtualization(void) { - cpu_emergency_unregister_virt_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); + x86_virt_unregister_emergency_callback(kvm_x86_ops.emergency_disable_virtualization_cpu); } int kvm_arch_enable_virtualization_cpu(void) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index a088b8eb3f14..8ee546718dfe 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -2,18 +2,54 @@ #include #include #include - #include +#include #include #include #include #include - #include +#include #include +struct x86_virt_ops { + int feature; + void (*emergency_disable_virtualization_cpu)(void); +}; +static struct x86_virt_ops virt_ops __ro_after_init; + __visible bool virt_rebooting; EXPORT_SYMBOL_FOR_KVM(virt_rebooting); +static cpu_emergency_virt_cb __rcu *kvm_emergency_callback; + +void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback))) + return; + + rcu_assign_pointer(kvm_emergency_callback, callback); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_register_emergency_callback); + +void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback) != callback)) + return; + + rcu_assign_pointer(kvm_emergency_callback, NULL); + synchronize_rcu(); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_unregister_emergency_callback); + +static void x86_virt_invoke_kvm_emergency_callback(void) +{ + cpu_emergency_virt_cb *kvm_callback; + + kvm_callback = rcu_dereference(kvm_emergency_callback); + if (kvm_callback) + kvm_callback(); +} + #if IS_ENABLED(CONFIG_KVM_INTEL) static DEFINE_PER_CPU(struct vmcs *, root_vmcs); @@ -42,6 +78,9 @@ int x86_vmx_enable_virtualization_cpu(void) { int r; + if (virt_ops.feature != X86_FEATURE_VMX) + return -EOPNOTSUPP; + if (cr4_read_shadow() & X86_CR4_VMXE) return -EBUSY; @@ -82,22 +121,24 @@ int x86_vmx_disable_virtualization_cpu(void) } EXPORT_SYMBOL_FOR_KVM(x86_vmx_disable_virtualization_cpu); -void x86_vmx_emergency_disable_virtualization_cpu(void) +static void x86_vmx_emergency_disable_virtualization_cpu(void) { virt_rebooting = true; /* * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be * set in task context. If this races with _another_ emergency call - * from NMI context, VMXOFF may #UD, but kernel will eat those faults - * due to virt_rebooting being set by the interrupting NMI callback. + * from NMI context, VMCLEAR (in KVM) and VMXOFF may #UD, but KVM and + * the kernel will eat those faults due to virt_rebooting being set by + * the interrupting NMI callback. */ if (!(__read_cr4() & X86_CR4_VMXE)) return; + x86_virt_invoke_kvm_emergency_callback(); + x86_vmx_disable_virtualization_cpu(); } -EXPORT_SYMBOL_FOR_KVM(x86_vmx_emergency_disable_virtualization_cpu); static __init void x86_vmx_exit(void) { @@ -111,6 +152,11 @@ static __init void x86_vmx_exit(void) static __init int __x86_vmx_init(void) { + const struct x86_virt_ops vmx_ops = { + .feature = X86_FEATURE_VMX, + .emergency_disable_virtualization_cpu = x86_vmx_emergency_disable_virtualization_cpu, + }; + u64 basic_msr; u32 rev_id; int cpu; @@ -147,6 +193,7 @@ static __init int __x86_vmx_init(void) per_cpu(root_vmcs, cpu) = vmcs; } + memcpy(&virt_ops, &vmx_ops, sizeof(virt_ops)); return 0; } @@ -161,6 +208,7 @@ static __init int x86_vmx_init(void) } #else static __init int x86_vmx_init(void) { return -EOPNOTSUPP; } +static __init void x86_vmx_exit(void) { } #endif #if IS_ENABLED(CONFIG_KVM_AMD) @@ -168,7 +216,7 @@ int x86_svm_enable_virtualization_cpu(void) { u64 efer; - if (!cpu_feature_enabled(X86_FEATURE_SVM)) + if (virt_ops.feature != X86_FEATURE_SVM) return -EOPNOTSUPP; rdmsrl(MSR_EFER, efer); @@ -201,7 +249,7 @@ int x86_svm_disable_virtualization_cpu(void) } EXPORT_SYMBOL_FOR_KVM(x86_svm_disable_virtualization_cpu); -void x86_svm_emergency_disable_virtualization_cpu(void) +static void x86_svm_emergency_disable_virtualization_cpu(void) { u64 efer; @@ -211,12 +259,68 @@ void x86_svm_emergency_disable_virtualization_cpu(void) if (!(efer & EFER_SVME)) return; + x86_virt_invoke_kvm_emergency_callback(); + x86_svm_disable_virtualization_cpu(); } -EXPORT_SYMBOL_FOR_KVM(x86_svm_emergency_disable_virtualization_cpu); + +static __init int x86_svm_init(void) +{ + const struct x86_virt_ops svm_ops = { + .feature = X86_FEATURE_SVM, + .emergency_disable_virtualization_cpu = x86_svm_emergency_disable_virtualization_cpu, + }; + + if (!cpu_feature_enabled(X86_FEATURE_SVM)) + return -EOPNOTSUPP; + + memcpy(&virt_ops, &svm_ops, sizeof(virt_ops)); + return 0; +} +#else +static __init int x86_svm_init(void) { return -EOPNOTSUPP; } #endif +/* + * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during + * reboot. VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if + * GIF=0, i.e. if the crash occurred between CLGI and STGI. + */ +int x86_virt_emergency_disable_virtualization_cpu(void) +{ + /* Ensure the !feature check can't get false positives. */ + BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX); + + if (!virt_ops.feature) + return -EOPNOTSUPP; + + /* + * IRQs must be disabled as virtualization is enabled in hardware via + * function call IPIs, i.e. IRQs need to be disabled to guarantee + * virtualization stays disabled. + */ + lockdep_assert_irqs_disabled(); + + /* + * Do the NMI shootdown even if virtualization is off on _this_ CPU, as + * other CPUs may have virtualization enabled. + */ + virt_ops.emergency_disable_virtualization_cpu(); + return 0; +} + void __init x86_virt_init(void) { - x86_vmx_init(); + /* + * Attempt to initialize both SVM and VMX, and simply use whichever one + * is present. Rsefuse to enable/use SVM or VMX if both are somehow + * supported. No known CPU supports both SVM and VMX. + */ + bool has_vmx = !x86_vmx_init(); + bool has_svm = !x86_svm_init(); + + if (WARN_ON_ONCE(has_vmx && has_svm)) { + x86_vmx_exit(); + memset(&virt_ops, 0, sizeof(virt_ops)); + } } -- Gitee From f582819cac77cd4b3963b01bcfa397c2b1fdfa4c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:55 -0800 Subject: [PATCH 27/78] x86/virt: Add refcounting of VMX/SVM usage to support multiple in-kernel users ANBZ: #40036 commit 8528a7f9c91d917ad2b3b6a71f1cb7e00b1fb1bf upstream. Implement a per-CPU refcounting scheme so that "users" of hardware virtualization, e.g. KVM and the future TDX code, can co-exist without pulling the rug out from under each other. E.g. if KVM were to disable VMX on module unload or when the last KVM VM was destroyed, SEAMCALLs from the TDX subsystem would #UD and panic the kernel. Disable preemption in the get/put APIs to ensure virtualization is fully enabled/disabled before returning to the caller. E.g. if the task were preempted after a 0=>1 transition, the new task would see a 1=>2 and thus return without enabling virtualization. Explicitly disable preemption instead of requiring the caller to do so, because the need to disable preemption is an artifact of the implementation. E.g. from KVM's perspective there is no _need_ to disable preemption as KVM guarantees the pCPU on which it is running is stable (but preemption is enabled). Opportunistically abstract away SVM vs. VMX in the public APIs by using X86_FEATURE_{SVM,VMX} to communicate what technology the caller wants to enable and use. backport: drop: svm_ops arch/x86/kvm/svm/svm.c.rej Intel-SIG: commit 8528a7f9c91d x86/virt: Add refcounting of VMX/SVM usage to support multiple in-kernel users Cc: Xu Yilun Reviewed-by: Chao Gao Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-10-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/virt.h | 6 ++--- arch/x86/kvm/vmx/vmx.c | 4 +-- arch/x86/virt/hw.c | 53 ++++++++++++++++++++++++++++++------- 3 files changed, 47 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h index 2c35534437e0..094c09b4f9e9 100644 --- a/arch/x86/include/asm/virt.h +++ b/arch/x86/include/asm/virt.h @@ -11,10 +11,8 @@ extern bool virt_rebooting; void __init x86_virt_init(void); -#if IS_ENABLED(CONFIG_KVM_INTEL) -int x86_vmx_enable_virtualization_cpu(void); -int x86_vmx_disable_virtualization_cpu(void); -#endif +int x86_virt_get_ref(int feat); +void x86_virt_put_ref(int feat); #if IS_ENABLED(CONFIG_KVM_AMD) int x86_svm_enable_virtualization_cpu(void); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index ace9eee202aa..b2142cc16534 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2791,7 +2791,7 @@ int vmx_enable_virtualization_cpu(void) if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu)) return -EFAULT; - return x86_vmx_enable_virtualization_cpu(); + return x86_virt_get_ref(X86_FEATURE_VMX); } static void vmclear_local_loaded_vmcss(void) @@ -2808,7 +2808,7 @@ void vmx_disable_virtualization_cpu(void) { vmclear_local_loaded_vmcss(); - x86_vmx_disable_virtualization_cpu(); + x86_virt_put_ref(X86_FEATURE_VMX); hv_reset_evmcs(); } diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 8ee546718dfe..6b7ff8f85000 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -13,6 +13,8 @@ struct x86_virt_ops { int feature; + int (*enable_virtualization_cpu)(void); + int (*disable_virtualization_cpu)(void); void (*emergency_disable_virtualization_cpu)(void); }; static struct x86_virt_ops virt_ops __ro_after_init; @@ -20,6 +22,8 @@ static struct x86_virt_ops virt_ops __ro_after_init; __visible bool virt_rebooting; EXPORT_SYMBOL_FOR_KVM(virt_rebooting); +static DEFINE_PER_CPU(int, virtualization_nr_users); + static cpu_emergency_virt_cb __rcu *kvm_emergency_callback; void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback) @@ -74,13 +78,10 @@ static int x86_virt_cpu_vmxon(void) return -EFAULT; } -int x86_vmx_enable_virtualization_cpu(void) +static int x86_vmx_enable_virtualization_cpu(void) { int r; - if (virt_ops.feature != X86_FEATURE_VMX) - return -EOPNOTSUPP; - if (cr4_read_shadow() & X86_CR4_VMXE) return -EBUSY; @@ -94,7 +95,6 @@ int x86_vmx_enable_virtualization_cpu(void) return 0; } -EXPORT_SYMBOL_FOR_KVM(x86_vmx_enable_virtualization_cpu); /* * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) @@ -105,7 +105,7 @@ EXPORT_SYMBOL_FOR_KVM(x86_vmx_enable_virtualization_cpu); * faults are guaranteed to be due to the !post-VMXON check unless the CPU is * magically in RM, VM86, compat mode, or at CPL>0. */ -int x86_vmx_disable_virtualization_cpu(void) +static int x86_vmx_disable_virtualization_cpu(void) { int r = -EIO; @@ -119,7 +119,6 @@ int x86_vmx_disable_virtualization_cpu(void) intel_pt_handle_vmx(0); return r; } -EXPORT_SYMBOL_FOR_KVM(x86_vmx_disable_virtualization_cpu); static void x86_vmx_emergency_disable_virtualization_cpu(void) { @@ -154,6 +153,8 @@ static __init int __x86_vmx_init(void) { const struct x86_virt_ops vmx_ops = { .feature = X86_FEATURE_VMX, + .enable_virtualization_cpu = x86_vmx_enable_virtualization_cpu, + .disable_virtualization_cpu = x86_vmx_disable_virtualization_cpu, .emergency_disable_virtualization_cpu = x86_vmx_emergency_disable_virtualization_cpu, }; @@ -281,6 +282,41 @@ static __init int x86_svm_init(void) static __init int x86_svm_init(void) { return -EOPNOTSUPP; } #endif +int x86_virt_get_ref(int feat) +{ + int r; + + /* Ensure the !feature check can't get false positives. */ + BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX); + + if (!virt_ops.feature || virt_ops.feature != feat) + return -EOPNOTSUPP; + + guard(preempt)(); + + if (this_cpu_inc_return(virtualization_nr_users) > 1) + return 0; + + r = virt_ops.enable_virtualization_cpu(); + if (r) + WARN_ON_ONCE(this_cpu_dec_return(virtualization_nr_users)); + + return r; +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_get_ref); + +void x86_virt_put_ref(int feat) +{ + guard(preempt)(); + + if (WARN_ON_ONCE(!this_cpu_read(virtualization_nr_users)) || + this_cpu_dec_return(virtualization_nr_users)) + return; + + BUG_ON(virt_ops.disable_virtualization_cpu() && !virt_rebooting); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_put_ref); + /* * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during * reboot. VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if @@ -288,9 +324,6 @@ static __init int x86_svm_init(void) { return -EOPNOTSUPP; } */ int x86_virt_emergency_disable_virtualization_cpu(void) { - /* Ensure the !feature check can't get false positives. */ - BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX); - if (!virt_ops.feature) return -EOPNOTSUPP; -- Gitee From 3f61312d742085c1802a4397c3d230c62d76eb6b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:56 -0800 Subject: [PATCH 28/78] x86/virt/tdx: Drop the outdated requirement that TDX be enabled in IRQ context ANBZ: #40036 commit 0efe5dc16169b0c7d47cbb495225065c67712fbc upstream. Remove TDX's outdated requirement that per-CPU enabling be done via IPI function call, which was a stale artifact leftover from early versions of the TDX enablement series. The requirement that IRQs be disabled should have been dropped as part of the revamped series that relied on a the KVM rework to enable VMX at module load. In other words, the kernel's "requirement" was never a requirement at all, but instead a reflection of how KVM enabled VMX (via IPI callback) when the TDX subsystem code was merged. Note, accessing per-CPU information is safe even without disabling IRQs, as tdx_online_cpu() is invoked via a cpuhp callback, i.e. from a per-CPU thread. Intel-SIG: commit 0efe5dc16169 x86/virt/tdx: Drop the outdated requirement that TDX be enabled in IRQ context Link: https://lore.kernel.org/all/ZyJOiPQnBz31qLZ7@google.com Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-11-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/tdx.c | 9 +-------- arch/x86/virt/vmx/tdx/tdx.c | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 855f3e4622cb..84ee42135f10 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3320,17 +3320,10 @@ int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) static int tdx_online_cpu(unsigned int cpu) { - unsigned long flags; - int r; - /* Sanity check CPU is already in post-VMXON */ WARN_ON_ONCE(!(cr4_read_shadow() & X86_CR4_VMXE)); - local_irq_save(flags); - r = tdx_cpu_enable(); - local_irq_restore(flags); - - return r; + return tdx_cpu_enable(); } static int tdx_offline_cpu(unsigned int cpu) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 27098c134df6..778280798fa6 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -106,8 +106,7 @@ static __always_inline int sc_retry_prerr(sc_func_t func, /* * Do the module global initialization once and return its result. - * It can be done on any cpu. It's always called with interrupts - * disabled. + * It can be done on any cpu, and from task or IRQ context. */ static int try_init_module_global(void) { @@ -116,8 +115,6 @@ static int try_init_module_global(void) static bool sysinit_done; static int sysinit_ret; - lockdep_assert_irqs_disabled(); - raw_spin_lock(&sysinit_lock); if (sysinit_done) @@ -148,8 +145,6 @@ static int try_init_module_global(void) * global initialization SEAMCALL if not done) on local cpu to make this * cpu be ready to run any other SEAMCALLs. * - * Always call this function via IPI function calls. - * * Return 0 on success, otherwise errors. */ int tdx_cpu_enable(void) @@ -160,8 +155,6 @@ int tdx_cpu_enable(void) if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM)) return -ENODEV; - lockdep_assert_irqs_disabled(); - if (__this_cpu_read(tdx_lp_initialized)) return 0; -- Gitee From b64b3fd24106e8925510953c4031a69faa585944 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:57 -0800 Subject: [PATCH 29/78] KVM: x86/tdx: Do VMXON and TDX-Module initialization during subsys init ANBZ: #40036 commit 165e77353831a85caa0444d16f29bd6b111dd2c5 upstream. Now that VMXON can be done without bouncing through KVM, do TDX-Module initialization during subsys init (specifically before module_init() so that it runs before KVM when both are built-in). Aside from the obvious benefits of separating core TDX code from KVM, this will allow tagging a pile of TDX functions and globals as being __init and __ro_after_init. backport: tdx_suspend(void *ign) --> tdx_suspend(void) struct syscore -->struct syscore_ops register_syscore(&tdx_syscore) --> register_syscore_ops(&tdx_syscore_ops) Intel-SIG: commit 165e77353831 KVM: x86/tdx: Do VMXON and TDX-Module initialization during subsys init Reviewed-by: Dan Williams Reviewed-by: Chao Gao Acked-by: Dave Hansen Tested-by: Chao Gao Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-12-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/arch/x86/tdx.rst | 36 ++------ arch/x86/include/asm/tdx.h | 4 - arch/x86/kvm/vmx/tdx.c | 149 +++++++------------------------ arch/x86/virt/vmx/tdx/tdx.c | 157 ++++++++++++++++++--------------- arch/x86/virt/vmx/tdx/tdx.h | 8 -- 5 files changed, 124 insertions(+), 230 deletions(-) diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst index 719043cd8b46..628542521b89 100644 --- a/Documentation/arch/x86/tdx.rst +++ b/Documentation/arch/x86/tdx.rst @@ -60,44 +60,18 @@ Besides initializing the TDX module, a per-cpu initialization SEAMCALL must be done on one cpu before any other SEAMCALLs can be made on that cpu. -The kernel provides two functions, tdx_enable() and tdx_cpu_enable() to -allow the user of TDX to enable the TDX module and enable TDX on local -cpu respectively. - -Making SEAMCALL requires VMXON has been done on that CPU. Currently only -KVM implements VMXON. For now both tdx_enable() and tdx_cpu_enable() -don't do VMXON internally (not trivial), but depends on the caller to -guarantee that. - -To enable TDX, the caller of TDX should: 1) temporarily disable CPU -hotplug; 2) do VMXON and tdx_enable_cpu() on all online cpus; 3) call -tdx_enable(). For example:: - - cpus_read_lock(); - on_each_cpu(vmxon_and_tdx_cpu_enable()); - ret = tdx_enable(); - cpus_read_unlock(); - if (ret) - goto no_tdx; - // TDX is ready to use - -And the caller of TDX must guarantee the tdx_cpu_enable() has been -successfully done on any cpu before it wants to run any other SEAMCALL. -A typical usage is do both VMXON and tdx_cpu_enable() in CPU hotplug -online callback, and refuse to online if tdx_cpu_enable() fails. - User can consult dmesg to see whether the TDX module has been initialized. If the TDX module is initialized successfully, dmesg shows something like below:: [..] virt/tdx: 262668 KBs allocated for PAMT - [..] virt/tdx: module initialized + [..] virt/tdx: TDX-Module initialized If the TDX module failed to initialize, dmesg also shows it failed to initialize:: - [..] virt/tdx: module initialization failed ... + [..] virt/tdx: TDX-Module initialization failed ... TDX Interaction to Other Kernel Components ------------------------------------------ @@ -129,9 +103,9 @@ CPU Hotplug ~~~~~~~~~~~ TDX module requires the per-cpu initialization SEAMCALL must be done on -one cpu before any other SEAMCALLs can be made on that cpu. The kernel -provides tdx_cpu_enable() to let the user of TDX to do it when the user -wants to use a new cpu for TDX task. +one cpu before any other SEAMCALLs can be made on that cpu. The kernel, +via the CPU hotplug framework, performs the necessary initialization when +a CPU is first brought online. TDX doesn't support physical (ACPI) CPU hotplug. During machine boot, TDX verifies all boot-time present logical CPUs are TDX compatible before diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index e38ad073e343..6455f36e2d51 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -122,8 +122,6 @@ static __always_inline u64 sc_retry(sc_func_t func, u64 fn, #define seamcall(_fn, _args) sc_retry(__seamcall, (_fn), (_args)) #define seamcall_ret(_fn, _args) sc_retry(__seamcall_ret, (_fn), (_args)) #define seamcall_saved_ret(_fn, _args) sc_retry(__seamcall_saved_ret, (_fn), (_args)) -int tdx_cpu_enable(void); -int tdx_enable(void); const char *tdx_dump_mce_info(struct mce *m); const struct tdx_sys_info *tdx_get_sysinfo(void); @@ -196,8 +194,6 @@ u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td); u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page); #else static inline void tdx_init(void) { } -static inline int tdx_cpu_enable(void) { return -ENODEV; } -static inline int tdx_enable(void) { return -ENODEV; } static inline u32 tdx_get_nr_guest_keyids(void) { return 0; } static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; } static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 84ee42135f10..5ca7cf93abf0 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -59,7 +59,7 @@ module_param_named(tdx, enable_tdx, bool, 0444); #define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) #define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) -static enum cpuhp_state tdx_cpuhp_state; +static enum cpuhp_state tdx_cpuhp_state __ro_after_init; static const struct tdx_sys_info *tdx_sysinfo; @@ -3320,10 +3320,7 @@ int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) static int tdx_online_cpu(unsigned int cpu) { - /* Sanity check CPU is already in post-VMXON */ - WARN_ON_ONCE(!(cr4_read_shadow() & X86_CR4_VMXE)); - - return tdx_cpu_enable(); + return 0; } static int tdx_offline_cpu(unsigned int cpu) @@ -3362,51 +3359,6 @@ static int tdx_offline_cpu(unsigned int cpu) return -EBUSY; } -static void __do_tdx_cleanup(void) -{ - /* - * Once TDX module is initialized, it cannot be disabled and - * re-initialized again w/o runtime update (which isn't - * supported by kernel). Only need to remove the cpuhp here. - * The TDX host core code tracks TDX status and can handle - * 'multiple enabling' scenario. - */ - WARN_ON_ONCE(!tdx_cpuhp_state); - cpuhp_remove_state_nocalls_cpuslocked(tdx_cpuhp_state); - tdx_cpuhp_state = 0; -} - -static void __tdx_cleanup(void) -{ - cpus_read_lock(); - __do_tdx_cleanup(); - cpus_read_unlock(); -} - -static int __init __do_tdx_bringup(void) -{ - int r; - - /* - * TDX-specific cpuhp callback to call tdx_cpu_enable() on all - * online CPUs before calling tdx_enable(), and on any new - * going-online CPU to make sure it is ready for TDX guest. - */ - r = cpuhp_setup_state_cpuslocked(CPUHP_AP_ONLINE_DYN, - "kvm/cpu/tdx:online", - tdx_online_cpu, tdx_offline_cpu); - if (r < 0) - return r; - - tdx_cpuhp_state = r; - - r = tdx_enable(); - if (r) - __do_tdx_cleanup(); - - return r; -} - static int __init __tdx_bringup(void) { const struct tdx_sys_info_td_conf *td_conf; @@ -3426,34 +3378,18 @@ static int __init __tdx_bringup(void) } } - /* - * Enabling TDX requires enabling hardware virtualization first, - * as making SEAMCALLs requires CPU being in post-VMXON state. - */ - r = kvm_enable_virtualization(); - if (r) - return r; - - cpus_read_lock(); - r = __do_tdx_bringup(); - cpus_read_unlock(); - - if (r) - goto tdx_bringup_err; - - r = -EINVAL; /* Get TDX global information for later use */ tdx_sysinfo = tdx_get_sysinfo(); - if (WARN_ON_ONCE(!tdx_sysinfo)) - goto get_sysinfo_err; + if (!tdx_sysinfo) + return -ENODEV; /* Check TDX module and KVM capabilities */ if (!tdx_get_supported_attrs(&tdx_sysinfo->td_conf) || !tdx_get_supported_xfam(&tdx_sysinfo->td_conf)) - goto get_sysinfo_err; + return -EINVAL; if (!(tdx_sysinfo->features.tdx_features0 & MD_FIELD_ID_FEATURES0_TOPOLOGY_ENUM)) - goto get_sysinfo_err; + return -EINVAL; /* * TDX has its own limit of maximum vCPUs it can support for all @@ -3488,34 +3424,32 @@ static int __init __tdx_bringup(void) if (td_conf->max_vcpus_per_td < num_present_cpus()) { pr_err("Disable TDX: MAX_VCPU_PER_TD (%u) smaller than number of logical CPUs (%u).\n", td_conf->max_vcpus_per_td, num_present_cpus()); - goto get_sysinfo_err; + return -EINVAL; } if (misc_cg_set_capacity(MISC_CG_RES_TDX, tdx_get_nr_guest_keyids())) - goto get_sysinfo_err; + return -EINVAL; /* - * Leave hardware virtualization enabled after TDX is enabled - * successfully. TDX CPU hotplug depends on this. + * TDX-specific cpuhp callback to disallow offlining the last CPU in a + * packing while KVM is running one or more TDs. Reclaiming HKIDs + * requires doing PAGE.WBINVD on every package, i.e. offlining all CPUs + * of a package would prevent reclaiming the HKID. */ + r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "kvm/cpu/tdx:online", + tdx_online_cpu, tdx_offline_cpu); + if (r < 0) + goto err_cpuhup; + + tdx_cpuhp_state = r; + return 0; -get_sysinfo_err: - __tdx_cleanup(); -tdx_bringup_err: - kvm_disable_virtualization(); +err_cpuhup: + misc_cg_set_capacity(MISC_CG_RES_TDX, 0); return r; } -void tdx_cleanup(void) -{ - if (enable_tdx) { - misc_cg_set_capacity(MISC_CG_RES_TDX, 0); - __tdx_cleanup(); - kvm_disable_virtualization(); - } -} - int __init tdx_bringup(void) { int r, i; @@ -3547,39 +3481,11 @@ int __init tdx_bringup(void) goto success_disable_tdx; } - if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { - pr_err("tdx: MOVDIR64B is required for TDX\n"); - goto success_disable_tdx; - } - - if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) { - pr_err("Self-snoop is required for TDX\n"); - goto success_disable_tdx; - } - if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) { - pr_err("tdx: no TDX private KeyIDs available\n"); + pr_err("TDX not supported by the host platform\n"); goto success_disable_tdx; } - if (!enable_virt_at_load) { - pr_err("tdx: tdx requires kvm.enable_virt_at_load=1\n"); - goto success_disable_tdx; - } - - /* - * Ideally KVM should probe whether TDX module has been loaded - * first and then try to bring it up. But TDX needs to use SEAMCALL - * to probe whether the module is loaded (there is no CPUID or MSR - * for that), and making SEAMCALL requires enabling virtualization - * first, just like the rest steps of bringing up TDX module. - * - * So, for simplicity do everything in __tdx_bringup(); the first - * SEAMCALL will return -ENODEV when the module is not loaded. The - * only complication is having to make sure that initialization - * SEAMCALLs don't return TDX_SEAMCALL_VMFAILINVALID in other - * cases. - */ r = __tdx_bringup(); if (r) { /* @@ -3594,8 +3500,6 @@ int __init tdx_bringup(void) */ if (r == -ENODEV) goto success_disable_tdx; - - enable_tdx = 0; } return r; @@ -3605,6 +3509,15 @@ int __init tdx_bringup(void) return 0; } +void tdx_cleanup(void) +{ + if (!enable_tdx) + return; + + misc_cg_set_capacity(MISC_CG_RES_TDX, 0); + cpuhp_remove_state(tdx_cpuhp_state); +} + void __init tdx_hardware_setup(void) { /* diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 778280798fa6..2c823186e659 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include #include #include +#include #include "tdx.h" static u32 tdx_global_keyid __ro_after_init; @@ -51,13 +53,11 @@ static DEFINE_PER_CPU(bool, tdx_lp_initialized); static struct tdmr_info_list tdx_tdmr_list; -static enum tdx_module_status_t tdx_module_status; -static DEFINE_MUTEX(tdx_module_lock); - /* All TDX-usable memory regions. Protected by mem_hotplug_lock. */ static LIST_HEAD(tdx_memlist); static struct tdx_sys_info tdx_sysinfo; +static bool tdx_module_initialized; typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); @@ -139,22 +139,15 @@ static int try_init_module_global(void) } /** - * tdx_cpu_enable - Enable TDX on local cpu - * - * Do one-time TDX module per-cpu initialization SEAMCALL (and TDX module - * global initialization SEAMCALL if not done) on local cpu to make this - * cpu be ready to run any other SEAMCALLs. - * - * Return 0 on success, otherwise errors. + * Enable VMXON and then do one-time TDX module per-cpu initialization SEAMCALL + * (and TDX module global initialization SEAMCALL if not done) on local cpu to + * make this cpu be ready to run any other SEAMCALLs. */ -int tdx_cpu_enable(void) +static int tdx_cpu_enable(void) { struct tdx_module_args args = {}; int ret; - if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM)) - return -ENODEV; - if (__this_cpu_read(tdx_lp_initialized)) return 0; @@ -175,7 +168,54 @@ int tdx_cpu_enable(void) return 0; } -EXPORT_SYMBOL_FOR_KVM(tdx_cpu_enable); + +static int tdx_online_cpu(unsigned int cpu) +{ + int ret; + + ret = x86_virt_get_ref(X86_FEATURE_VMX); + if (ret) + return ret; + + ret = tdx_cpu_enable(); + if (ret) + x86_virt_put_ref(X86_FEATURE_VMX); + + return ret; +} + +static int tdx_offline_cpu(unsigned int cpu) +{ + x86_virt_put_ref(X86_FEATURE_VMX); + return 0; +} + +static void tdx_shutdown_cpu(void *ign) +{ + x86_virt_put_ref(X86_FEATURE_VMX); +} + +static void tdx_shutdown(void) +{ + on_each_cpu(tdx_shutdown_cpu, NULL, 1); +} + +static int tdx_suspend(void) +{ + x86_virt_put_ref(X86_FEATURE_VMX); + return 0; +} + +static void tdx_resume(void) +{ + WARN_ON_ONCE(x86_virt_get_ref(X86_FEATURE_VMX)); +} + +static struct syscore_ops tdx_syscore_ops = { + .suspend = tdx_suspend, + .resume = tdx_resume, + .shutdown = tdx_shutdown, +}; /* * Add a memory region as a TDX memory block. The caller must make sure @@ -1148,67 +1188,51 @@ static int init_tdx_module(void) goto out_put_tdxmem; } -static int __tdx_enable(void) +static int tdx_enable(void) { + enum cpuhp_state state; int ret; - ret = init_tdx_module(); - if (ret) { - pr_err("module initialization failed (%d)\n", ret); - tdx_module_status = TDX_MODULE_ERROR; - return ret; + if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) { + pr_err("TDX not supported by the host platform\n"); + return -ENODEV; } - pr_info("module initialized\n"); - tdx_module_status = TDX_MODULE_INITIALIZED; + if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) { + pr_err("XSAVE is required for TDX\n"); + return -EINVAL; + } - return 0; -} + if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { + pr_err("MOVDIR64B is required for TDX\n"); + return -EINVAL; + } -/** - * tdx_enable - Enable TDX module to make it ready to run TDX guests - * - * This function assumes the caller has: 1) held read lock of CPU hotplug - * lock to prevent any new cpu from becoming online; 2) done both VMXON - * and tdx_cpu_enable() on all online cpus. - * - * This function requires there's at least one online cpu for each CPU - * package to succeed. - * - * This function can be called in parallel by multiple callers. - * - * Return 0 if TDX is enabled successfully, otherwise error. - */ -int tdx_enable(void) -{ - int ret; - if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM)) + if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) { + pr_err("Self-snoop is required for TDX\n"); return -ENODEV; + } - lockdep_assert_cpus_held(); - - mutex_lock(&tdx_module_lock); + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "virt/tdx:online", + tdx_online_cpu, tdx_offline_cpu); + if (state < 0) + return state; - switch (tdx_module_status) { - case TDX_MODULE_UNINITIALIZED: - ret = __tdx_enable(); - break; - case TDX_MODULE_INITIALIZED: - /* Already initialized, great, tell the caller. */ - ret = 0; - break; - default: - /* Failed to initialize in the previous attempts */ - ret = -EINVAL; - break; + ret = init_tdx_module(); + if (ret) { + pr_err("TDX-Module initialization failed (%d)\n", ret); + cpuhp_remove_state(state); + return ret; } - mutex_unlock(&tdx_module_lock); + register_syscore_ops(&tdx_syscore_ops); - return ret; + tdx_module_initialized = true; + pr_info("TDX-Module initialized\n"); + return 0; } -EXPORT_SYMBOL_FOR_KVM(tdx_enable); +subsys_initcall(tdx_enable); static bool is_pamt_page(unsigned long phys) { @@ -1459,15 +1483,10 @@ void __init tdx_init(void) const struct tdx_sys_info *tdx_get_sysinfo(void) { - const struct tdx_sys_info *p = NULL; - - /* Make sure all fields in @tdx_sysinfo have been populated */ - mutex_lock(&tdx_module_lock); - if (tdx_module_status == TDX_MODULE_INITIALIZED) - p = (const struct tdx_sys_info *)&tdx_sysinfo; - mutex_unlock(&tdx_module_lock); + if (!tdx_module_initialized) + return NULL; - return p; + return (const struct tdx_sys_info *)&tdx_sysinfo; } EXPORT_SYMBOL_FOR_KVM(tdx_get_sysinfo); diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index 82bb82be8567..dde219c823b4 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -91,14 +91,6 @@ struct tdmr_info { * Do not put any hardware-defined TDX structure representations below * this comment! */ - -/* Kernel defined TDX module status during module initialization. */ -enum tdx_module_status_t { - TDX_MODULE_UNINITIALIZED, - TDX_MODULE_INITIALIZED, - TDX_MODULE_ERROR -}; - struct tdx_memblock { struct list_head list; unsigned long start_pfn; -- Gitee From 45d81a36a6cb8206753fb0ca949fa6f790498110 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Tue, 19 Aug 2025 18:58:10 +0300 Subject: [PATCH 30/78] x86/tdx: Tidy reset_pamt functions ANBZ: #40036 commit a27b008a5d7e8c49740dfd4b560cd2d1abe722e4 upstream. tdx_quirk_reset_paddr() was renamed to reflect that, in fact, the clearing is necessary only for hardware with a certain quirk. That is dealt with in a subsequent patch. Rename reset_pamt functions to contain "quirk" to reflect the new functionality, and remove the now misleading comment. Intel-SIG: commit a27b008a5d7e x86/tdx: Tidy reset_pamt functions Signed-off-by: Adrian Hunter Signed-off-by: Dave Hansen Reviewed-by: Rick Edgecombe Reviewed-by: Binbin Wu Acked-by: Kai Huang Acked-by: Vishal Annapurve Link: https://lore.kernel.org/all/20250819155811.136099-3-adrian.hunter%40intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 2c823186e659..3aa7d97298e0 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -688,17 +688,17 @@ static void reset_tdx_pages(unsigned long base, unsigned long size) mb(); } -static void tdmr_reset_pamt(struct tdmr_info *tdmr) +static void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) { tdmr_do_pamt_func(tdmr, reset_tdx_pages); } -static void tdmrs_reset_pamt_all(struct tdmr_info_list *tdmr_list) +static void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) { int i; for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) - tdmr_reset_pamt(tdmr_entry(tdmr_list, i)); + tdmr_quirk_reset_pamt(tdmr_entry(tdmr_list, i)); } static unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list) @@ -1170,15 +1170,7 @@ static int init_tdx_module(void) * to the kernel. */ wbinvd_on_all_cpus(); - /* - * According to the TDX hardware spec, if the platform - * doesn't have the "partial write machine check" - * erratum, any kernel read/write will never cause #MC - * in kernel space, thus it's OK to not convert PAMTs - * back to normal. But do the conversion anyway here - * as suggested by the TDX spec. - */ - tdmrs_reset_pamt_all(&tdx_tdmr_list); + tdmrs_quirk_reset_pamt_all(&tdx_tdmr_list); err_free_pamts: tdmrs_free_pamt_all(&tdx_tdmr_list); err_free_tdmrs: -- Gitee From 04fa5f782f3973f584e7f92fa068f96b9c596087 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:26:58 -0800 Subject: [PATCH 31/78] x86/virt/tdx: Tag a pile of functions as __init, and globals as __ro_after_init ANBZ: #40036 commit 9900400e20c0289bf0c82231169c33b43e38c6e8 upstream. Now that TDX-Module initialization is done during subsys init, tag all related functions as __init, and relevant data as __ro_after_init. backport: not these function: static void tdmrs_quirk_reset_pamt_all() static __init void tdmrs_quirk_reset_pamt_all() Intel-SIG: commit 9900400e20c0 x86/virt/tdx: Tag a pile of functions as __init, and globals as __ro_after_init Reviewed-by: Dan Williams Reviewed-by: Chao Gao Tested-by: Chao Gao Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-13-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 118 ++++++++++---------- arch/x86/virt/vmx/tdx/tdx_global_metadata.c | 10 +- 2 files changed, 65 insertions(+), 63 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 3aa7d97298e0..690ae60052b9 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -56,8 +56,8 @@ static struct tdmr_info_list tdx_tdmr_list; /* All TDX-usable memory regions. Protected by mem_hotplug_lock. */ static LIST_HEAD(tdx_memlist); -static struct tdx_sys_info tdx_sysinfo; -static bool tdx_module_initialized; +static struct tdx_sys_info tdx_sysinfo __ro_after_init; +static bool tdx_module_initialized __ro_after_init; typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); @@ -222,8 +222,9 @@ static struct syscore_ops tdx_syscore_ops = { * all memory regions are added in address ascending order and don't * overlap. */ -static int add_tdx_memblock(struct list_head *tmb_list, unsigned long start_pfn, - unsigned long end_pfn, int nid) +static __init int add_tdx_memblock(struct list_head *tmb_list, + unsigned long start_pfn, + unsigned long end_pfn, int nid) { struct tdx_memblock *tmb; @@ -241,7 +242,7 @@ static int add_tdx_memblock(struct list_head *tmb_list, unsigned long start_pfn, return 0; } -static void free_tdx_memlist(struct list_head *tmb_list) +static __init void free_tdx_memlist(struct list_head *tmb_list) { /* @tmb_list is protected by mem_hotplug_lock */ while (!list_empty(tmb_list)) { @@ -259,7 +260,7 @@ static void free_tdx_memlist(struct list_head *tmb_list) * ranges off in a secondary structure because memblock is modified * in memory hotplug while TDX memory regions are fixed. */ -static int build_tdx_memlist(struct list_head *tmb_list) +static __init int build_tdx_memlist(struct list_head *tmb_list) { unsigned long start_pfn, end_pfn; int i, nid, ret; @@ -291,7 +292,7 @@ static int build_tdx_memlist(struct list_head *tmb_list) return ret; } -static int read_sys_metadata_field(u64 field_id, u64 *data) +static __init int read_sys_metadata_field(u64 field_id, u64 *data) { struct tdx_module_args args = {}; int ret; @@ -313,7 +314,7 @@ static int read_sys_metadata_field(u64 field_id, u64 *data) #include "tdx_global_metadata.c" -static int check_features(struct tdx_sys_info *sysinfo) +static __init int check_features(struct tdx_sys_info *sysinfo) { u64 tdx_features0 = sysinfo->features.tdx_features0; @@ -326,7 +327,7 @@ static int check_features(struct tdx_sys_info *sysinfo) } /* Calculate the actual TDMR size */ -static int tdmr_size_single(u16 max_reserved_per_tdmr) +static __init int tdmr_size_single(u16 max_reserved_per_tdmr) { int tdmr_sz; @@ -340,8 +341,8 @@ static int tdmr_size_single(u16 max_reserved_per_tdmr) return ALIGN(tdmr_sz, TDMR_INFO_ALIGNMENT); } -static int alloc_tdmr_list(struct tdmr_info_list *tdmr_list, - struct tdx_sys_info_tdmr *sysinfo_tdmr) +static __init int alloc_tdmr_list(struct tdmr_info_list *tdmr_list, + struct tdx_sys_info_tdmr *sysinfo_tdmr) { size_t tdmr_sz, tdmr_array_sz; void *tdmr_array; @@ -372,7 +373,7 @@ static int alloc_tdmr_list(struct tdmr_info_list *tdmr_list, return 0; } -static void free_tdmr_list(struct tdmr_info_list *tdmr_list) +static __init void free_tdmr_list(struct tdmr_info_list *tdmr_list) { free_pages_exact(tdmr_list->tdmrs, tdmr_list->max_tdmrs * tdmr_list->tdmr_sz); @@ -401,8 +402,8 @@ static inline u64 tdmr_end(struct tdmr_info *tdmr) * preallocated @tdmr_list, following all the special alignment * and size rules for TDMR. */ -static int fill_out_tdmrs(struct list_head *tmb_list, - struct tdmr_info_list *tdmr_list) +static __init int fill_out_tdmrs(struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list) { struct tdx_memblock *tmb; int tdmr_idx = 0; @@ -478,8 +479,8 @@ static int fill_out_tdmrs(struct list_head *tmb_list, * Calculate PAMT size given a TDMR and a page size. The returned * PAMT size is always aligned up to 4K page boundary. */ -static unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz, - u16 pamt_entry_size) +static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz, + u16 pamt_entry_size) { unsigned long pamt_sz, nr_pamt_entries; @@ -510,7 +511,7 @@ static unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz, * PAMT. This node will have some memory covered by the TDMR. The * relative amount of memory covered is not considered. */ -static int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list) +static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list) { struct tdx_memblock *tmb; @@ -539,9 +540,9 @@ static int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list) * Allocate PAMTs from the local NUMA node of some memory in @tmb_list * within @tdmr, and set up PAMTs for @tdmr. */ -static int tdmr_set_up_pamt(struct tdmr_info *tdmr, - struct list_head *tmb_list, - u16 pamt_entry_size[]) +static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr, + struct list_head *tmb_list, + u16 pamt_entry_size[]) { unsigned long pamt_base[TDX_PS_NR]; unsigned long pamt_size[TDX_PS_NR]; @@ -611,7 +612,7 @@ static void tdmr_get_pamt(struct tdmr_info *tdmr, unsigned long *pamt_base, *pamt_size = pamt_sz; } -static void tdmr_do_pamt_func(struct tdmr_info *tdmr, +static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr, void (*pamt_func)(unsigned long base, unsigned long size)) { unsigned long pamt_base, pamt_size; @@ -628,17 +629,17 @@ static void tdmr_do_pamt_func(struct tdmr_info *tdmr, pamt_func(pamt_base, pamt_size); } -static void free_pamt(unsigned long pamt_base, unsigned long pamt_size) +static __init void free_pamt(unsigned long pamt_base, unsigned long pamt_size) { free_contig_range(pamt_base >> PAGE_SHIFT, pamt_size >> PAGE_SHIFT); } -static void tdmr_free_pamt(struct tdmr_info *tdmr) +static __init void tdmr_free_pamt(struct tdmr_info *tdmr) { tdmr_do_pamt_func(tdmr, free_pamt); } -static void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list) +static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list) { int i; @@ -647,9 +648,9 @@ static void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list) } /* Allocate and set up PAMTs for all TDMRs */ -static int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, - struct list_head *tmb_list, - u16 pamt_entry_size[]) +static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, + struct list_head *tmb_list, + u16 pamt_entry_size[]) { int i, ret = 0; @@ -688,12 +689,12 @@ static void reset_tdx_pages(unsigned long base, unsigned long size) mb(); } -static void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) +static __init void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) { tdmr_do_pamt_func(tdmr, reset_tdx_pages); } -static void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) +static __init void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) { int i; @@ -701,7 +702,7 @@ static void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) tdmr_quirk_reset_pamt(tdmr_entry(tdmr_list, i)); } -static unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list) +static __init unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list) { unsigned long pamt_size = 0; int i; @@ -716,8 +717,8 @@ static unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list) return pamt_size / 1024; } -static int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, u64 addr, - u64 size, u16 max_reserved_per_tdmr) +static __init int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, + u64 addr, u64 size, u16 max_reserved_per_tdmr) { struct tdmr_reserved_area *rsvd_areas = tdmr->reserved_areas; int idx = *p_idx; @@ -750,10 +751,10 @@ static int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, u64 addr, * those holes fall within @tdmr, set up a TDMR reserved area to cover * the hole. */ -static int tdmr_populate_rsvd_holes(struct list_head *tmb_list, - struct tdmr_info *tdmr, - int *rsvd_idx, - u16 max_reserved_per_tdmr) +static __init int tdmr_populate_rsvd_holes(struct list_head *tmb_list, + struct tdmr_info *tdmr, + int *rsvd_idx, + u16 max_reserved_per_tdmr) { struct tdx_memblock *tmb; u64 prev_end; @@ -814,10 +815,10 @@ static int tdmr_populate_rsvd_holes(struct list_head *tmb_list, * overlaps with @tdmr, set up a TDMR reserved area to cover the * overlapping part. */ -static int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list, - struct tdmr_info *tdmr, - int *rsvd_idx, - u16 max_reserved_per_tdmr) +static __init int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list, + struct tdmr_info *tdmr, + int *rsvd_idx, + u16 max_reserved_per_tdmr) { int i, ret; @@ -852,7 +853,7 @@ static int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list, } /* Compare function called by sort() for TDMR reserved areas */ -static int rsvd_area_cmp_func(const void *a, const void *b) +static __init int rsvd_area_cmp_func(const void *a, const void *b) { struct tdmr_reserved_area *r1 = (struct tdmr_reserved_area *)a; struct tdmr_reserved_area *r2 = (struct tdmr_reserved_area *)b; @@ -871,10 +872,10 @@ static int rsvd_area_cmp_func(const void *a, const void *b) * Populate reserved areas for the given @tdmr, including memory holes * (via @tmb_list) and PAMTs (via @tdmr_list). */ -static int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr, - struct list_head *tmb_list, - struct tdmr_info_list *tdmr_list, - u16 max_reserved_per_tdmr) +static __init int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr, + struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list, + u16 max_reserved_per_tdmr) { int ret, rsvd_idx = 0; @@ -899,9 +900,9 @@ static int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr, * Populate reserved areas for all TDMRs in @tdmr_list, including memory * holes (via @tmb_list) and PAMTs. */ -static int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list, - struct list_head *tmb_list, - u16 max_reserved_per_tdmr) +static __init int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list, + struct list_head *tmb_list, + u16 max_reserved_per_tdmr) { int i; @@ -922,9 +923,9 @@ static int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list, * to cover all TDX memory regions in @tmb_list based on the TDX module * TDMR global information in @sysinfo_tdmr. */ -static int construct_tdmrs(struct list_head *tmb_list, - struct tdmr_info_list *tdmr_list, - struct tdx_sys_info_tdmr *sysinfo_tdmr) +static __init int construct_tdmrs(struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list, + struct tdx_sys_info_tdmr *sysinfo_tdmr) { u16 pamt_entry_size[TDX_PS_NR] = { sysinfo_tdmr->pamt_4k_entry_size, @@ -956,7 +957,8 @@ static int construct_tdmrs(struct list_head *tmb_list, return ret; } -static int config_tdx_module(struct tdmr_info_list *tdmr_list, u64 global_keyid) +static __init int config_tdx_module(struct tdmr_info_list *tdmr_list, + u64 global_keyid) { struct tdx_module_args args = {}; u64 *tdmr_pa_array; @@ -991,7 +993,7 @@ static int config_tdx_module(struct tdmr_info_list *tdmr_list, u64 global_keyid) return ret; } -static int do_global_key_config(void *unused) +static __init int do_global_key_config(void *unused) { struct tdx_module_args args = {}; @@ -1009,7 +1011,7 @@ static int do_global_key_config(void *unused) * KVM) can ensure success by ensuring sufficient CPUs are online and * can run SEAMCALLs. */ -static int config_global_keyid(void) +static __init int config_global_keyid(void) { cpumask_var_t packages; int cpu, ret = -EINVAL; @@ -1049,7 +1051,7 @@ static int config_global_keyid(void) return ret; } -static int init_tdmr(struct tdmr_info *tdmr) +static __init int init_tdmr(struct tdmr_info *tdmr) { u64 next; @@ -1080,7 +1082,7 @@ static int init_tdmr(struct tdmr_info *tdmr) return 0; } -static int init_tdmrs(struct tdmr_info_list *tdmr_list) +static __init int init_tdmrs(struct tdmr_info_list *tdmr_list) { int i; @@ -1099,7 +1101,7 @@ static int init_tdmrs(struct tdmr_info_list *tdmr_list) return 0; } -static int init_tdx_module(void) +static __init int init_tdx_module(void) { int ret; @@ -1180,7 +1182,7 @@ static int init_tdx_module(void) goto out_put_tdxmem; } -static int tdx_enable(void) +static __init int tdx_enable(void) { enum cpuhp_state state; int ret; diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c index 4c9917a9c2c3..e854aa0ca649 100644 --- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -22,7 +22,7 @@ static int get_tdx_sys_info_version(struct tdx_sys_info_version *sysinfo_version return ret; } -static int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_features) +static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_features) { int ret = 0; u64 val; @@ -33,7 +33,7 @@ static int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_featu return ret; } -static int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) +static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) { int ret = 0; u64 val; @@ -52,7 +52,7 @@ static int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) return ret; } -static int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl) +static __init int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl) { int ret = 0; u64 val; @@ -67,7 +67,7 @@ static int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl return ret; } -static int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf) +static __init int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf) { int ret = 0; u64 val; @@ -100,7 +100,7 @@ static int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf return ret; } -static int get_tdx_sys_info(struct tdx_sys_info *sysinfo) +static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo) { int ret = 0; -- Gitee From 44a944fe9012f3e6d1ff84f71c1895d7bab6385f Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Fri, 13 Feb 2026 17:26:59 -0800 Subject: [PATCH 32/78] x86/virt/tdx: KVM: Consolidate TDX CPU hotplug handling ANBZ: #40036 commit eac90a5ba0aa40f6d81def241bd78d2a5cc5e08b upstream. The core kernel registers a CPU hotplug callback to do VMX and TDX init and deinit while KVM registers a separate CPU offline callback to block offlining the last online CPU in a socket. Splitting TDX-related CPU hotplug handling across two components is odd and adds unnecessary complexity. Consolidate TDX-related CPU hotplug handling by integrating KVM's tdx_offline_cpu() to the one in the core kernel. Also move nr_configured_hkid to the core kernel because tdx_offline_cpu() references it. Since HKID allocation and free are handled in the core kernel, it's more natural to track used HKIDs there. Intel-SIG: commit eac90a5ba0aa x86/virt/tdx: KVM: Consolidate TDX CPU hotplug handling Reviewed-by: Dan Williams Signed-off-by: Chao Gao Tested-by: Chao Gao Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-14-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/tdx.c | 68 +------------------------------------ arch/x86/virt/vmx/tdx/tdx.c | 49 ++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 70 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 5ca7cf93abf0..c56588a64a2d 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -59,8 +59,6 @@ module_param_named(tdx, enable_tdx, bool, 0444); #define TDX_SHARED_BIT_PWL_5 gpa_to_gfn(BIT_ULL(51)) #define TDX_SHARED_BIT_PWL_4 gpa_to_gfn(BIT_ULL(47)) -static enum cpuhp_state tdx_cpuhp_state __ro_after_init; - static const struct tdx_sys_info *tdx_sysinfo; void tdh_vp_rd_failed(struct vcpu_tdx *tdx, char *uclass, u32 field, u64 err) @@ -219,8 +217,6 @@ static int init_kvm_tdx_caps(const struct tdx_sys_info_td_conf *td_conf, */ static DEFINE_MUTEX(tdx_lock); -static atomic_t nr_configured_hkid; - static bool tdx_operand_busy(u64 err) { return (err & TDX_SEAMCALL_STATUS_MASK) == TDX_OPERAND_BUSY; @@ -268,7 +264,6 @@ static inline void tdx_hkid_free(struct kvm_tdx *kvm_tdx) { tdx_guest_keyid_free(kvm_tdx->hkid); kvm_tdx->hkid = -1; - atomic_dec(&nr_configured_hkid); misc_cg_uncharge(MISC_CG_RES_TDX, kvm_tdx->misc_cg, 1); put_misc_cg(kvm_tdx->misc_cg); kvm_tdx->misc_cg = NULL; @@ -2423,8 +2418,6 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params, ret = -ENOMEM; - atomic_inc(&nr_configured_hkid); - tdr_page = alloc_page(GFP_KERNEL_ACCOUNT); if (!tdr_page) goto free_hkid; @@ -3318,51 +3311,10 @@ int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) return PG_LEVEL_4K; } -static int tdx_online_cpu(unsigned int cpu) -{ - return 0; -} - -static int tdx_offline_cpu(unsigned int cpu) -{ - int i; - - /* No TD is running. Allow any cpu to be offline. */ - if (!atomic_read(&nr_configured_hkid)) - return 0; - - /* - * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to - * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory - * controller with pconfig. If we have active TDX HKID, refuse to - * offline the last online cpu. - */ - for_each_online_cpu(i) { - /* - * Found another online cpu on the same package. - * Allow to offline. - */ - if (i != cpu && topology_physical_package_id(i) == - topology_physical_package_id(cpu)) - return 0; - } - - /* - * This is the last cpu of this package. Don't offline it. - * - * Because it's hard for human operator to understand the - * reason, warn it. - */ -#define MSG_ALLPKG_ONLINE \ - "TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n" - pr_warn_ratelimited(MSG_ALLPKG_ONLINE); - return -EBUSY; -} - static int __init __tdx_bringup(void) { const struct tdx_sys_info_td_conf *td_conf; - int r, i; + int i; for (i = 0; i < ARRAY_SIZE(tdx_uret_msrs); i++) { /* @@ -3430,24 +3382,7 @@ static int __init __tdx_bringup(void) if (misc_cg_set_capacity(MISC_CG_RES_TDX, tdx_get_nr_guest_keyids())) return -EINVAL; - /* - * TDX-specific cpuhp callback to disallow offlining the last CPU in a - * packing while KVM is running one or more TDs. Reclaiming HKIDs - * requires doing PAGE.WBINVD on every package, i.e. offlining all CPUs - * of a package would prevent reclaiming the HKID. - */ - r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "kvm/cpu/tdx:online", - tdx_online_cpu, tdx_offline_cpu); - if (r < 0) - goto err_cpuhup; - - tdx_cpuhp_state = r; - return 0; - -err_cpuhup: - misc_cg_set_capacity(MISC_CG_RES_TDX, 0); - return r; } int __init tdx_bringup(void) @@ -3515,7 +3450,6 @@ void tdx_cleanup(void) return; misc_cg_set_capacity(MISC_CG_RES_TDX, 0); - cpuhp_remove_state(tdx_cpuhp_state); } void __init tdx_hardware_setup(void) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 690ae60052b9..9ab43305441d 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -59,6 +59,8 @@ static LIST_HEAD(tdx_memlist); static struct tdx_sys_info tdx_sysinfo __ro_after_init; static bool tdx_module_initialized __ro_after_init; +static atomic_t nr_configured_hkid; + typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args) @@ -186,6 +188,40 @@ static int tdx_online_cpu(unsigned int cpu) static int tdx_offline_cpu(unsigned int cpu) { + int i; + + /* No TD is running. Allow any cpu to be offline. */ + if (!atomic_read(&nr_configured_hkid)) + goto done; + + /* + * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to + * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory + * controller with pconfig. If we have active TDX HKID, refuse to + * offline the last online cpu. + */ + for_each_online_cpu(i) { + /* + * Found another online cpu on the same package. + * Allow to offline. + */ + if (i != cpu && topology_physical_package_id(i) == + topology_physical_package_id(cpu)) + goto done; + } + + /* + * This is the last cpu of this package. Don't offline it. + * + * Because it's hard for human operator to understand the + * reason, warn it. + */ +#define MSG_ALLPKG_ONLINE \ + "TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n" + pr_warn_ratelimited(MSG_ALLPKG_ONLINE); + return -EBUSY; + +done: x86_virt_put_ref(X86_FEATURE_VMX); return 0; } @@ -1492,15 +1528,22 @@ EXPORT_SYMBOL_FOR_KVM(tdx_get_nr_guest_keyids); int tdx_guest_keyid_alloc(void) { - return ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start, - tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, - GFP_KERNEL); + int ret; + + ret = ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start, + tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, + GFP_KERNEL); + if (ret >= 0) + atomic_inc(&nr_configured_hkid); + + return ret; } EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_alloc); void tdx_guest_keyid_free(unsigned int keyid) { ida_free(&tdx_guest_keyid_pool, keyid); + atomic_dec(&nr_configured_hkid); } EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_free); -- Gitee From c46b487fce1e9069a03f1dea50feceefe0d292e8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:27:00 -0800 Subject: [PATCH 33/78] x86/virt/tdx: Use ida_is_empty() to detect if any TDs may be running ANBZ: #40036 commit afe31de159bf218d6e92db6a4495f715f0a4e38c upstream. Drop nr_configured_hkid and instead use ida_is_empty() to detect if any HKIDs have been allocated/configured. Intel-SIG: commit afe31de159bf x86/virt/tdx: Use ida_is_empty() to detect if any TDs may be running Suggested-by: Dan Williams Reviewed-by: Dan Williams Reviewed-by: Chao Gao Tested-by: Chao Gao Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-15-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 9ab43305441d..6e1bdb5632b6 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -59,8 +59,6 @@ static LIST_HEAD(tdx_memlist); static struct tdx_sys_info tdx_sysinfo __ro_after_init; static bool tdx_module_initialized __ro_after_init; -static atomic_t nr_configured_hkid; - typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args) @@ -191,7 +189,7 @@ static int tdx_offline_cpu(unsigned int cpu) int i; /* No TD is running. Allow any cpu to be offline. */ - if (!atomic_read(&nr_configured_hkid)) + if (ida_is_empty(&tdx_guest_keyid_pool)) goto done; /* @@ -1528,22 +1526,15 @@ EXPORT_SYMBOL_FOR_KVM(tdx_get_nr_guest_keyids); int tdx_guest_keyid_alloc(void) { - int ret; - - ret = ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start, - tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, - GFP_KERNEL); - if (ret >= 0) - atomic_inc(&nr_configured_hkid); - - return ret; + return ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start, + tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, + GFP_KERNEL); } EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_alloc); void tdx_guest_keyid_free(unsigned int keyid) { ida_free(&tdx_guest_keyid_pool, keyid); - atomic_dec(&nr_configured_hkid); } EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_free); -- Gitee From 18126bc06ada247c5d0b37948adb08e672f66520 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:27:01 -0800 Subject: [PATCH 34/78] KVM: Bury kvm_{en,dis}able_virtualization() in kvm_main.c once more ANBZ: #40036 commit d30372d0b7e637475c79a785d055f4eb8c863656 upstream. Now that TDX handles doing VMXON without KVM's involvement, bury the top-level APIs to enable and disable virtualization back in kvm_main.c. No functional change intended. Intel-SIG: commit d30372d0b7e6 KVM: Bury kvm_{en,dis}able_virtualization() in kvm_main.c once more Reviewed-by: Dan Williams Reviewed-by: Chao Gao Tested-by: Chao Gao Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-16-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- include/linux/kvm_host.h | 8 -------- virt/kvm/kvm_main.c | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 9085e3e3b5f9..fdb90587e22d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2654,12 +2654,4 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif -#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING -int kvm_enable_virtualization(void); -void kvm_disable_virtualization(void); -#else -static inline int kvm_enable_virtualization(void) { return 0; } -static inline void kvm_disable_virtualization(void) { } -#endif - #endif diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index ceb2c4c802df..6c0e7add164f 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1232,6 +1232,9 @@ void __weak kvm_arch_create_vm_debugfs(struct kvm *kvm) { } +static int kvm_enable_virtualization(void); +static void kvm_disable_virtualization(void); + static struct kvm *kvm_create_vm(unsigned long type, const char *fdname) { struct kvm *kvm = kvm_arch_alloc_vm(); @@ -5857,7 +5860,7 @@ static struct syscore_ops kvm_syscore_ops = { .shutdown = kvm_shutdown, }; -int kvm_enable_virtualization(void) +static int kvm_enable_virtualization(void) { int r; @@ -5902,9 +5905,8 @@ int kvm_enable_virtualization(void) --kvm_usage_count; return r; } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_virtualization); -void kvm_disable_virtualization(void) +static void kvm_disable_virtualization(void) { guard(mutex)(&kvm_usage_lock); @@ -5915,7 +5917,6 @@ void kvm_disable_virtualization(void) cpuhp_remove_state(CPUHP_AP_KVM_ONLINE); kvm_arch_disable_virtualization(); } -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_disable_virtualization); static int kvm_init_virtualization(void) { @@ -5931,6 +5932,14 @@ static void kvm_uninit_virtualization(void) kvm_disable_virtualization(); } #else /* CONFIG_KVM_GENERIC_HARDWARE_ENABLING */ +static int kvm_enable_virtualization(void) +{ + return 0; +} +static void kvm_disable_virtualization(void) +{ + +} static int kvm_init_virtualization(void) { return 0; -- Gitee From 224d5d0f30181ffe42d4ee92a4cf113621c10905 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Feb 2026 17:27:02 -0800 Subject: [PATCH 35/78] KVM: TDX: Fold tdx_bringup() into tdx_hardware_setup() ANBZ: #40036 commit f630de1f8d70d7e29e12bc25dc63f9c5f771dc59 upstream. Now that TDX doesn't need to manually enable virtualization through _KVM_ APIs during setup, fold tdx_bringup() into tdx_hardware_setup() where the code belongs, e.g. so that KVM doesn't leave the S-EPT kvm_x86_ops wired up when TDX is disabled. The weird ordering (and naming) was necessary to allow KVM TDX to use kvm_enable_virtualization(), which in turn had a hard dependency on kvm_x86_ops.enable_virtualization_cpu and thus kvm_x86_vendor_init(). Intel-SIG: commit f630de1f8d70 KVM: TDX: Fold tdx_bringup() into tdx_hardware_setup() Tested-by: Chao Gao Reviewed-by: Dan Williams Tested-by: Sagi Shahar Link: https://patch.msgid.link/20260214012702.2368778-17-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/main.c | 19 ++++++++----------- arch/x86/kvm/vmx/tdx.c | 35 +++++++++++++++-------------------- arch/x86/kvm/vmx/tdx.h | 8 ++------ 3 files changed, 25 insertions(+), 37 deletions(-) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 725f9163c498..03c05a048557 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -29,10 +29,15 @@ static __init int vt_hardware_setup(void) if (ret) return ret; + return enable_tdx ? tdx_hardware_setup() : 0; +} + +static void vt_hardware_unsetup(void) +{ if (enable_tdx) - tdx_hardware_setup(); + tdx_hardware_unsetup(); - return 0; + vmx_hardware_unsetup(); } static int vt_vm_init(struct kvm *kvm) @@ -861,7 +866,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .check_processor_compatibility = vmx_check_processor_compat, - .hardware_unsetup = vmx_hardware_unsetup, + .hardware_unsetup = vt_op(hardware_unsetup), .enable_virtualization_cpu = vmx_enable_virtualization_cpu, .disable_virtualization_cpu = vt_op(disable_virtualization_cpu), @@ -1024,7 +1029,6 @@ struct kvm_x86_init_ops vt_init_ops __initdata = { static void __exit vt_exit(void) { kvm_exit(); - tdx_cleanup(); vmx_exit(); } module_exit(vt_exit); @@ -1038,11 +1042,6 @@ static int __init vt_init(void) if (r) return r; - /* tdx_init() has been taken */ - r = tdx_bringup(); - if (r) - goto err_tdx_bringup; - /* * TDX and VMX have different vCPU structures. Calculate the * maximum size/align so that kvm_init() can use the larger @@ -1069,8 +1068,6 @@ static int __init vt_init(void) return 0; err_kvm_init: - tdx_cleanup(); -err_tdx_bringup: vmx_exit(); return r; } diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index c56588a64a2d..67b4e926b5fd 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3311,7 +3311,12 @@ int tdx_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) return PG_LEVEL_4K; } -static int __init __tdx_bringup(void) +void tdx_hardware_unsetup(void) +{ + misc_cg_set_capacity(MISC_CG_RES_TDX, 0); +} + +static int __init __tdx_hardware_setup(void) { const struct tdx_sys_info_td_conf *td_conf; int i; @@ -3385,7 +3390,7 @@ static int __init __tdx_bringup(void) return 0; } -int __init tdx_bringup(void) +int __init tdx_hardware_setup(void) { int r, i; @@ -3421,7 +3426,7 @@ int __init tdx_bringup(void) goto success_disable_tdx; } - r = __tdx_bringup(); + r = __tdx_hardware_setup(); if (r) { /* * Disable TDX only but don't fail to load module if the TDX @@ -3435,25 +3440,10 @@ int __init tdx_bringup(void) */ if (r == -ENODEV) goto success_disable_tdx; - } - - return r; - -success_disable_tdx: - enable_tdx = 0; - return 0; -} - -void tdx_cleanup(void) -{ - if (!enable_tdx) - return; - misc_cg_set_capacity(MISC_CG_RES_TDX, 0); -} + return r; + } -void __init tdx_hardware_setup(void) -{ /* * Note, if the TDX module can't be loaded, KVM TDX support will be * disabled but KVM will continue loading (see tdx_bringup()). @@ -3465,4 +3455,9 @@ void __init tdx_hardware_setup(void) vt_x86_ops.free_external_spt = tdx_sept_free_private_spt; vt_x86_ops.remove_external_spte = tdx_sept_remove_private_spte; vt_x86_ops.protected_apic_has_interrupt = tdx_protected_apic_has_interrupt; + return 0; + +success_disable_tdx: + enable_tdx = 0; + return 0; } diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h index 45b5183ccb36..b5cd2ffb303e 100644 --- a/arch/x86/kvm/vmx/tdx.h +++ b/arch/x86/kvm/vmx/tdx.h @@ -8,9 +8,8 @@ #ifdef CONFIG_KVM_INTEL_TDX #include "common.h" -void tdx_hardware_setup(void); -int tdx_bringup(void); -void tdx_cleanup(void); +int tdx_hardware_setup(void); +void tdx_hardware_unsetup(void); extern bool enable_tdx; @@ -187,9 +186,6 @@ TDX_BUILD_TDVPS_ACCESSORS(8, MANAGEMENT, management); TDX_BUILD_TDVPS_ACCESSORS(64, STATE_NON_ARCH, state_non_arch); #else -static inline int tdx_bringup(void) { return 0; } -static inline void tdx_cleanup(void) {} - #define enable_tdx 0 struct kvm_tdx { -- Gitee From 186a03984245a75c3d567e48d739848dda986b96 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Thu, 2 Apr 2026 00:32:01 -0600 Subject: [PATCH 36/78] x86/tdx: Move TDX architectural error codes into ANBZ: #40036 commit 53642715861e838f328a3fbef99a1d315955221a upstream. Today there are two separate locations where TDX error codes are defined: arch/x86/include/asm/tdx.h arch/x86/kvm/vmx/tdx_errno.h They have some overlap that is already defined similarly. Reduce the duplication by unifying the architectural error codes at: asm/shared/tdx_errno.h ...and update the headers that contained the duplicated definitions to include the new unified header. "asm/shared" is used for sharing TDX code between the early compressed code and the normal kernel code. While the compressed code for the guest doesn't use these error code header definitions today, it does make the types of calls that return the values they define. So place the defines in "shared" location so that it can, but leave such cleanups for future changes. Intel-SIG: commit 53642715861e x86/tdx: Move TDX architectural error codes into [Rick: enhance log] [Vishal: reduce to a simple move of architectural defines only] Signed-off-by: Kirill A. Shutemov Signed-off-by: Rick Edgecombe Signed-off-by: Vishal Verma Signed-off-by: Dave Hansen Reviewed-by: Chao Gao Acked-by: Sean Christopherson Link: https://patch.msgid.link/20260402-fuller_tdx_kexec_support-v3-1-34438d7094bf@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/shared/tdx.h | 1 + arch/x86/{kvm/vmx => include/asm/shared}/tdx_errno.h | 7 +++---- arch/x86/kvm/vmx/tdx.h | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) rename arch/x86/{kvm/vmx => include/asm/shared}/tdx_errno.h (92%) diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h index 6bb150c52bc0..f200a5f30329 100644 --- a/arch/x86/include/asm/shared/tdx.h +++ b/arch/x86/include/asm/shared/tdx.h @@ -4,6 +4,7 @@ #include #include +#include #define TDX_HYPERCALL_STANDARD 0 diff --git a/arch/x86/kvm/vmx/tdx_errno.h b/arch/x86/include/asm/shared/tdx_errno.h similarity index 92% rename from arch/x86/kvm/vmx/tdx_errno.h rename to arch/x86/include/asm/shared/tdx_errno.h index 6ff4672c4181..3c1e8ce716e3 100644 --- a/arch/x86/kvm/vmx/tdx_errno.h +++ b/arch/x86/include/asm/shared/tdx_errno.h @@ -1,8 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* architectural status code for SEAMCALL */ - -#ifndef __KVM_X86_TDX_ERRNO_H -#define __KVM_X86_TDX_ERRNO_H +#ifndef _ASM_X86_SHARED_TDX_ERRNO_H +#define _ASM_X86_SHARED_TDX_ERRNO_H #define TDX_SEAMCALL_STATUS_MASK 0xFFFFFFFF00000000ULL @@ -37,4 +36,4 @@ #define TDX_OPERAND_ID_SEPT 0x92 #define TDX_OPERAND_ID_TD_EPOCH 0xa9 -#endif /* __KVM_X86_TDX_ERRNO_H */ +#endif /* _ASM_X86_SHARED_TDX_ERRNO_H */ diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h index b5cd2ffb303e..ac8323a68b16 100644 --- a/arch/x86/kvm/vmx/tdx.h +++ b/arch/x86/kvm/vmx/tdx.h @@ -3,7 +3,6 @@ #define __KVM_X86_VMX_TDX_H #include "tdx_arch.h" -#include "tdx_errno.h" #ifdef CONFIG_KVM_INTEL_TDX #include "common.h" -- Gitee From 189770987427392e051dbd7570c243acfc35c9e8 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Thu, 2 Apr 2026 00:32:03 -0600 Subject: [PATCH 37/78] x86/virt/tdx: Add SEAMCALL wrapper for TDH.SYS.DISABLE ANBZ: #40036 commit b7d2173946efa20434aefd7421b46a90f1080fbe upstream. Some early TDX-capable platforms have an erratum where a partial write to TDX private memory can cause a machine check on a subsequent read. On these platforms, kexec and kdump have been disabled in these cases, because the old kernel cannot safely hand off TDX state to the new kernel. Later TDX modules support the TDH.SYS.DISABLE SEAMCALL, which provides a way to cleanly disable TDX and allow kexec to proceed. The new SEAMCALL has an enumeration bit, but that is ignored. It is expected that users will be using the latest TDX module, and the failure mode for running the missing SEAMCALL on an older module is not fatal. This can be a long running operation, and the time needed largely depends on the amount of memory that has been allocated to TDs. If all TDs have been destroyed prior to the sys_disable call, then it is fast, with only needing to override the TDX module memory. After the SEAMCALL completes, the TDX module is disabled and all memory resources allocated to TDX are freed and reset. The next kernel can then re-initialize the TDX module from scratch via the normal TDX bring-up sequence. The SEAMCALL can return two different error codes that expect a retry. - TDX_INTERRUPTED_RESUMABLE can be returned in the case of a host interrupt. However, it will not return until it makes some forward progress, so we can expect to complete even in the case of interrupt storms. - TDX_SYS_BUSY will be returned on contention with other TDH.SYS.* SEAMCALLs, however a side effect of TDH.SYS.DISABLE is that it will block other SEAMCALLs once it gets going. So this contention will be short lived. So loop infinitely on either of these error codes, until success or other error. An error is printed if the SEAMCALL fails with anything other than the error codes that cause retries, or 'synthesized' error codes produced for #GP or #UD. e.g., an old module that has been properly initialized, that doesn't implement SYS_DISABLE, returns TDX_OPERAND_INVALID. This prints: virt/tdx: TDH.SYS.DISABLE failed: 0xc000010000000000 But a system that doesn't have any TDX support at all doesn't print anything. Intel-SIG: commit b7d2173946ef x86/virt/tdx: Add SEAMCALL wrapper for TDH.SYS.DISABLE Co-developed-by: Rick Edgecombe Signed-off-by: Rick Edgecombe Signed-off-by: Vishal Verma Signed-off-by: Dave Hansen Reviewed-by: Chao Gao Reviewed-by: Kiryl Shutsemau (Meta) Acked-by: Kai Huang Link: https://patch.msgid.link/20260402-fuller_tdx_kexec_support-v3-3-34438d7094bf@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/shared/tdx_errno.h | 1 + arch/x86/include/asm/tdx.h | 3 +++ arch/x86/virt/vmx/tdx/tdx.c | 30 +++++++++++++++++++++++++ arch/x86/virt/vmx/tdx/tdx.h | 1 + 4 files changed, 35 insertions(+) diff --git a/arch/x86/include/asm/shared/tdx_errno.h b/arch/x86/include/asm/shared/tdx_errno.h index 3c1e8ce716e3..ee411b360e20 100644 --- a/arch/x86/include/asm/shared/tdx_errno.h +++ b/arch/x86/include/asm/shared/tdx_errno.h @@ -13,6 +13,7 @@ #define TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE 0x6000000500000000ULL #define TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE 0x6000000700000000ULL #define TDX_INTERRUPTED_RESUMABLE 0x8000000300000000ULL +#define TDX_SYS_BUSY 0x8000020200000000ULL #define TDX_OPERAND_INVALID 0xC000010000000000ULL #define TDX_OPERAND_BUSY 0x8000020000000000ULL #define TDX_PREVIOUS_TLB_EPOCH_BUSY 0x8000020100000000ULL diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 6455f36e2d51..d3ceb6ea9bc1 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -166,6 +166,8 @@ static inline int pg_level_to_tdx_sept_level(enum pg_level level) return level - 1; } +void tdx_sys_disable(void); + u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args); u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page); u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2); @@ -197,6 +199,7 @@ static inline void tdx_init(void) { } static inline u32 tdx_get_nr_guest_keyids(void) { return 0; } static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; } static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; } +static inline void tdx_sys_disable(void) { } #endif /* CONFIG_INTEL_TDX_HOST */ #endif /* !__ASSEMBLY__ */ diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 6e1bdb5632b6..d0781655c945 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1911,3 +1911,33 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page) return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); } EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid); + +void tdx_sys_disable(void) +{ + struct tdx_module_args args = {}; + u64 ret; + + /* + * Don't loop forever. + * + * - TDX_INTERRUPTED_RESUMABLE guarantees forward progress between + * calls. + * + * - TDX_SYS_BUSY could be returned due to contention with other + * TDH.SYS.* SEAMCALLs, but will lock out *new* TDH.SYS.* SEAMCALLs, + * so that SYS.DISABLE can eventually make progress. + * + * This is a 'destructive' SEAMCALL, in that no other SEAMCALL can be + * run after this until a full reinitialization is done. + */ + do { + ret = seamcall(TDH_SYS_DISABLE, &args); + } while (ret == TDX_INTERRUPTED_RESUMABLE || ret == TDX_SYS_BUSY); + + /* + * Print SEAMCALL failures, but not SW-defined error codes + * (SEAMCALL faulted with #GP/#UD, TDX not supported). + */ + if (ret && (ret & TDX_SW_ERROR) != TDX_SW_ERROR) + pr_err("TDH.SYS.DISABLE failed: 0x%016llx\n", ret); +} diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index dde219c823b4..e2cf2dd48755 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -46,6 +46,7 @@ #define TDH_PHYMEM_PAGE_WBINVD 41 #define TDH_VP_WR 43 #define TDH_SYS_CONFIG 45 +#define TDH_SYS_DISABLE 69 /* * SEAMCALL leaf: -- Gitee From 0df115422358dc2ec425535747b415e5cd778469 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:46 -0700 Subject: [PATCH 38/78] x86/virt/tdx: Clarify try_init_module_global() result caching ANBZ: #40036 commit 1ffa6a10253c417b281aff3cbd02bdf43b2b159d upstream. TDX module global initialization is executed only once. The first call caches both the return code and the "done" state in static function variables. Later callers read the variables. A lock protects the saved state and serializes callers. These variables will soon be moved to a global structure. Prepare for that by treating the variables as a unit. Assign them together and limit accesses to while the lock is held. Intel-SIG: commit 1ffa6a10253c x86/virt/tdx: Clarify try_init_module_global() result caching [ dhansen: mostly rewrite changelog ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-2-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index d0781655c945..f37c2e48b06d 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -114,28 +114,34 @@ static int try_init_module_global(void) static DEFINE_RAW_SPINLOCK(sysinit_lock); static bool sysinit_done; static int sysinit_ret; + int ret; raw_spin_lock(&sysinit_lock); - if (sysinit_done) + /* Return the "cached" return code. */ + if (sysinit_done) { + ret = sysinit_ret; goto out; + } /* RCX is module attributes and all bits are reserved */ args.rcx = 0; - sysinit_ret = seamcall_prerr(TDH_SYS_INIT, &args); + ret = seamcall_prerr(TDH_SYS_INIT, &args); /* * The first SEAMCALL also detects the TDX module, thus * it can fail due to the TDX module is not loaded. * Dump message to let the user know. */ - if (sysinit_ret == -ENODEV) + if (ret == -ENODEV) pr_err("module not loaded\n"); + /* Save the return code for later callers. */ sysinit_done = true; + sysinit_ret = ret; out: raw_spin_unlock(&sysinit_lock); - return sysinit_ret; + return ret; } /** -- Gitee From a98054efb8c754eb7ddc5f9d86c39dbd62ac48c3 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:47 -0700 Subject: [PATCH 39/78] x86/virt/tdx: Move TDX global initialization states to file scope ANBZ: #40036 commit 9c0c68708dc48e8ae4f9043c96b387a0173361f1 upstream. TDX module global initialization is executed only once. The first call caches both the result and the "done" state, and later callers reuse the saved result. A lock protects that cached states. Those states and the lock are currently kept as function-local statics because they are used only by try_init_module_global(). TDX module updates need to reset the cached states so TDX global initialization can be run again after an update. That will add another access site in the same file. Move the cached states to file scope so it is accessible outside try_init_module_global(), and move the lock along with the states it protects. No functional change intended. Intel-SIG: commit 9c0c68708dc4 x86/virt/tdx: Move TDX global initialization states to file scope Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-3-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index f37c2e48b06d..f1bae070da14 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -104,6 +104,10 @@ static __always_inline int sc_retry_prerr(sc_func_t func, #define seamcall_prerr_ret(__fn, __args) \ sc_retry_prerr(__seamcall_ret, seamcall_err_ret, (__fn), (__args)) +static DEFINE_RAW_SPINLOCK(sysinit_lock); +static bool sysinit_done; +static int sysinit_ret; + /* * Do the module global initialization once and return its result. * It can be done on any cpu, and from task or IRQ context. @@ -111,9 +115,6 @@ static __always_inline int sc_retry_prerr(sc_func_t func, static int try_init_module_global(void) { struct tdx_module_args args = {}; - static DEFINE_RAW_SPINLOCK(sysinit_lock); - static bool sysinit_done; - static int sysinit_ret; int ret; raw_spin_lock(&sysinit_lock); -- Gitee From c4cf75b2819f0f7760b92d4088b379a7ffe70bf2 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:48 -0700 Subject: [PATCH 40/78] x86/virt/tdx: Consolidate TDX global initialization states ANBZ: #40036 commit 451735bf90bfb294bb542cb2fa39ed01822aab86 upstream. The kernel uses several global flags to guard one-time TDX initialization flows and prevent them from being repeated. When the TDX module is updated, all of those states must be reset so that the module can be initialized again. Today those states are kept as separate global variables, which makes the reset path awkward and easy to miss when a new state is added. Group the states into a single structure so they can be reset together, for example with memset(), and so a newly added state won't be missed. Drop the __ro_after_init annotation from tdx_module_initialized because the other two states do not have it. And with TDX module update support, all the states need to be writable at runtime. Intel-SIG: commit 451735bf90bf x86/virt/tdx: Consolidate TDX global initialization states Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-4-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index f1bae070da14..2b19b2de44de 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -43,6 +43,13 @@ #include #include "tdx.h" +struct tdx_module_state { + bool initialized; + bool sysinit_done; + int sysinit_ret; +}; + +static struct tdx_module_state tdx_module_state; static u32 tdx_global_keyid __ro_after_init; static u32 tdx_guest_keyid_start __ro_after_init; static u32 tdx_nr_guest_keyids __ro_after_init; @@ -57,7 +64,6 @@ static struct tdmr_info_list tdx_tdmr_list; static LIST_HEAD(tdx_memlist); static struct tdx_sys_info tdx_sysinfo __ro_after_init; -static bool tdx_module_initialized __ro_after_init; typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); @@ -105,8 +111,6 @@ static __always_inline int sc_retry_prerr(sc_func_t func, sc_retry_prerr(__seamcall_ret, seamcall_err_ret, (__fn), (__args)) static DEFINE_RAW_SPINLOCK(sysinit_lock); -static bool sysinit_done; -static int sysinit_ret; /* * Do the module global initialization once and return its result. @@ -120,8 +124,8 @@ static int try_init_module_global(void) raw_spin_lock(&sysinit_lock); /* Return the "cached" return code. */ - if (sysinit_done) { - ret = sysinit_ret; + if (tdx_module_state.sysinit_done) { + ret = tdx_module_state.sysinit_ret; goto out; } @@ -138,8 +142,8 @@ static int try_init_module_global(void) pr_err("module not loaded\n"); /* Save the return code for later callers. */ - sysinit_done = true; - sysinit_ret = ret; + tdx_module_state.sysinit_done = true; + tdx_module_state.sysinit_ret = ret; out: raw_spin_unlock(&sysinit_lock); return ret; @@ -1263,7 +1267,7 @@ static __init int tdx_enable(void) register_syscore_ops(&tdx_syscore_ops); - tdx_module_initialized = true; + tdx_module_state.initialized = true; pr_info("TDX-Module initialized\n"); return 0; } @@ -1518,7 +1522,7 @@ void __init tdx_init(void) const struct tdx_sys_info *tdx_get_sysinfo(void) { - if (!tdx_module_initialized) + if (!tdx_module_state.initialized) return NULL; return (const struct tdx_sys_info *)&tdx_sysinfo; -- Gitee From b4b4fe110515ae6c5a3ce76df014c7a7b7933833 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Tue, 19 Aug 2025 18:58:09 +0300 Subject: [PATCH 41/78] x86/tdx: Eliminate duplicate code in tdx_clear_page() ANBZ: #40036 commit 94272b084a745940e076a170d8193ac3427292e6 upstream. tdx_clear_page() and reset_tdx_pages() duplicate the TDX page clearing logic. Rename reset_tdx_pages() to tdx_quirk_reset_paddr() and create tdx_quirk_reset_page() to call tdx_quirk_reset_paddr() and be used in place of tdx_clear_page(). The new name reflects that, in fact, the clearing is necessary only for hardware with a certain quirk. That is dealt with in a subsequent patch but doing the rename here avoids additional churn. Note reset_tdx_pages() is slightly different from tdx_clear_page() because, more appropriately, it uses mb() in place of __mb(). Except when extra debugging is enabled (kcsan at present), mb() just calls __mb(). Intel-SIG: commit 94272b084a74 x86/tdx: Eliminate duplicate code in tdx_clear_page() Signed-off-by: Adrian Hunter Signed-off-by: Dave Hansen Reviewed-by: Kirill A. Shutemov Reviewed-by: Binbin Wu Reviewed-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Acked-by: Kai Huang Acked-by: Sean Christopherson Acked-by: Vishal Annapurve Link: https://lore.kernel.org/all/20250819155811.136099-2-adrian.hunter%40intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx.h | 2 ++ arch/x86/kvm/vmx/tdx.c | 25 +++---------------------- arch/x86/virt/vmx/tdx/tdx.c | 10 ++++++++-- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index d3ceb6ea9bc1..c26d329bad38 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -129,6 +129,8 @@ int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); +void tdx_quirk_reset_page(struct page *page); + struct tdx_td { /* TD root structure: */ struct page *tdr_page; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 67b4e926b5fd..097ef17559a1 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -290,25 +290,6 @@ static inline void tdx_disassociate_vp(struct kvm_vcpu *vcpu) vcpu->cpu = -1; } -static void tdx_clear_page(struct page *page) -{ - const void *zero_page = (const void *) page_to_virt(ZERO_PAGE(0)); - void *dest = page_to_virt(page); - unsigned long i; - - /* - * The page could have been poisoned. MOVDIR64B also clears - * the poison bit so the kernel can safely use the page again. - */ - for (i = 0; i < PAGE_SIZE; i += 64) - movdir64b(dest + i, zero_page); - /* - * MOVDIR64B store uses WC buffer. Prevent following memory reads - * from seeing potentially poisoned cache. - */ - __mb(); -} - /* * Execute a SEAMCALL related to removing/blocking S-EPT entries, with a single * retry (if necessary) after forcing vCPUs to exit and wait for the operation @@ -362,7 +343,7 @@ static int tdx_reclaim_page(struct page *page) r = __tdx_reclaim_page(page); if (!r) - tdx_clear_page(page); + tdx_quirk_reset_page(page); return r; } @@ -606,7 +587,7 @@ static void tdx_reclaim_td_control_pages(struct kvm *kvm) if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_clear_page(kvm_tdx->td.tdr_page); + tdx_quirk_reset_page(kvm_tdx->td.tdr_page); __free_page(kvm_tdx->td.tdr_page); kvm_tdx->td.tdr_page = NULL; @@ -1842,7 +1823,7 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn, if (TDX_BUG_ON(err, TDH_PHYMEM_PAGE_WBINVD, kvm)) return; - tdx_clear_page(page); + tdx_quirk_reset_page(page); } void tdx_deliver_interrupt(struct kvm_lapic *apic, int delivery_mode, diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 2b19b2de44de..c900b471aef7 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -717,7 +717,7 @@ static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, * clear these pages. Note this function doesn't flush cache of * these TDX private pages. The caller should make sure of that. */ -static void reset_tdx_pages(unsigned long base, unsigned long size) +static void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) { const void *zero_page = (const void *)page_address(ZERO_PAGE(0)); unsigned long phys, end; @@ -734,9 +734,15 @@ static void reset_tdx_pages(unsigned long base, unsigned long size) mb(); } +void tdx_quirk_reset_page(struct page *page) +{ + tdx_quirk_reset_paddr(page_to_phys(page), PAGE_SIZE); +} +EXPORT_SYMBOL_GPL(tdx_quirk_reset_page); + static __init void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) { - tdmr_do_pamt_func(tdmr, reset_tdx_pages); + tdmr_do_pamt_func(tdmr, tdx_quirk_reset_paddr); } static __init void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) -- Gitee From 85cc4c38c97f8609f73b82c26adb0f74ac35e236 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:49 -0700 Subject: [PATCH 42/78] x86/virt/tdx: Move TDX_FEATURES0 bits to asm/tdx.h ANBZ: #40036 commit 77525820de70afd11ad7652bda005ce0ec1343af upstream. Future changes will add support for new TDX features exposed as TDX_FEATURES0 bits. The presence of these features will need to be checked outside of arch/x86/virt. The feature query helpers and the TDX_FEATURES0 defines they reference will need to live in the widely accessible asm/tdx.h header. Move the existing TDX_FEATURES0 to asm/tdx.h so that they can all be kept together. Opportunistically switch to BIT_ULL() since TDX_FEATURES0 is 64-bit. No functional change intended. Intel-SIG: commit 77525820de70 x86/virt/tdx: Move TDX_FEATURES0 bits to asm/tdx.h [ dhansen: grammar fixups ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Link: https://lore.kernel.org/kvm/20260427152854.101171-17-chao.gao@intel.com/ # [1] Link: https://lore.kernel.org/kvm/20251121005125.417831-16-rick.p.edgecombe@intel.com/ # [2] Link: https://patch.msgid.link/20260520133909.409394-5-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx.h | 3 +++ arch/x86/virt/vmx/tdx/tdx.h | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index c26d329bad38..c278263985b1 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -32,6 +32,9 @@ #define TDX_SUCCESS 0ULL #define TDX_RND_NO_ENTROPY 0x8000020300000000ULL +/* Bit definitions of TDX_FEATURES0 metadata field */ +#define TDX_FEATURES0_NO_RBP_MOD BIT_ULL(18) + #ifndef __ASSEMBLY__ #include diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index e2cf2dd48755..76c5fb1e1ffe 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -85,9 +85,6 @@ struct tdmr_info { DECLARE_FLEX_ARRAY(struct tdmr_reserved_area, reserved_areas); } __packed __aligned(TDMR_INFO_ALIGNMENT); -/* Bit definitions of TDX_FEATURES0 metadata field */ -#define TDX_FEATURES0_NO_RBP_MOD BIT(18) - /* * Do not put any hardware-defined TDX structure representations below * this comment! -- Gitee From 85625bcca1ad4e9678016a9c4df24f915eafa550 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Tue, 19 Aug 2025 18:58:11 +0300 Subject: [PATCH 43/78] x86/tdx: Skip clearing reclaimed pages unless X86_BUG_TDX_PW_MCE is present ANBZ: #40036 commit 01fb93a363e0583a3ce48098aca5ab9825a5b790 upstream. Avoid clearing reclaimed TDX private pages unless the platform is affected by the X86_BUG_TDX_PW_MCE erratum. This significantly reduces VM shutdown time on unaffected systems. Background KVM currently clears reclaimed TDX private pages using MOVDIR64B, which: - Clears the TD Owner bit (which identifies TDX private memory) and integrity metadata without triggering integrity violations. - Clears poison from cache lines without consuming it, avoiding MCEs on access (refer TDX Module Base spec. 1348549-006US section 6.5. Handling Machine Check Events during Guest TD Operation). The TDX module also uses MOVDIR64B to initialize private pages before use. If cache flushing is needed, it sets TDX_FEATURES.CLFLUSH_BEFORE_ALLOC. However, KVM currently flushes unconditionally, refer commit 94c477a751c7b ("x86/virt/tdx: Add SEAMCALL wrappers to add TD private pages") In contrast, when private pages are reclaimed, the TDX Module handles flushing via the TDH.PHYMEM.CACHE.WB SEAMCALL. Problem Clearing all private pages during VM shutdown is costly. For guests with a large amount of memory it can take minutes. Solution TDX Module Base Architecture spec. documents that private pages reclaimed from a TD should be initialized using MOVDIR64B, in order to avoid integrity violation or TD bit mismatch detection when later being read using a shared HKID, refer April 2025 spec. "Page Initialization" in section "8.6.2. Platforms not Using ACT: Required Cache Flush and Initialization by the Host VMM" That is an overstatement and will be clarified in coming versions of the spec. In fact, as outlined in "Table 16.2: Non-ACT Platforms Checks on Memory" and "Table 16.3: Non-ACT Platforms Checks on Memory Reads in Li Mode" in the same spec, there is no issue accessing such reclaimed pages using a shared key that does not have integrity enabled. Linux always uses KeyID 0 which never has integrity enabled. KeyID 0 is also the TME KeyID which disallows integrity, refer "TME Policy/Encryption Algorithm" bit description in "Intel Architecture Memory Encryption Technologies" spec version 1.6 April 2025. So there is no need to clear pages to avoid integrity violations. There remains a risk of poison consumption. However, in the context of TDX, it is expected that there would be a machine check associated with the original poisoning. On some platforms that results in a panic. However platforms may support "SEAM_NR" Machine Check capability, in which case Linux machine check handler marks the page as poisoned, which prevents it from being allocated anymore, refer commit 7911f145de5fe ("x86/mce: Implement recovery for errors in TDX/SEAM non-root mode") Improvement By skipping the clearing step on unaffected platforms, shutdown time can improve by up to 40%. On platforms with the X86_BUG_TDX_PW_MCE erratum (SPR and EMR), continue clearing because these platforms may trigger poison on partial writes to previously-private pages, even with KeyID 0, refer commit 1e536e1068970 ("x86/cpu: Detect TDX partial write machine check erratum") Intel-SIG: commit 01fb93a363e0 x86/tdx: Skip clearing reclaimed pages unless X86_BUG_TDX_PW_MCE is present Signed-off-by: Adrian Hunter Signed-off-by: Dave Hansen Reviewed-by: Kirill A. Shutemov Reviewed-by: Rick Edgecombe Reviewed-by: Xiaoyao Li Reviewed-by: Binbin Wu Acked-by: Kai Huang Acked-by: Vishal Annapurve Link: https://lore.kernel.org/all/20250819155811.136099-4-adrian.hunter%40intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index c900b471aef7..90930f5fd311 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -713,15 +713,19 @@ static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, } /* - * Convert TDX private pages back to normal by using MOVDIR64B to - * clear these pages. Note this function doesn't flush cache of - * these TDX private pages. The caller should make sure of that. + * Convert TDX private pages back to normal by using MOVDIR64B to clear these + * pages. Typically, any write to the page will convert it from TDX private back + * to normal kernel memory. Systems with the X86_BUG_TDX_PW_MCE erratum need to + * do the conversion explicitly via MOVDIR64B. */ static void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) { const void *zero_page = (const void *)page_address(ZERO_PAGE(0)); unsigned long phys, end; + if (!boot_cpu_has_bug(X86_BUG_TDX_PW_MCE)) + return; + end = base + size; for (phys = base; phys < end; phys += 64) movdir64b(__va(phys), zero_page); -- Gitee From 1179023c5532260f654ad833b9aafaf9a9ced73c Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 20 May 2026 15:28:51 -0700 Subject: [PATCH 44/78] x86/virt/tdx: Move low level SEAMCALL helpers out of ANBZ: #40036 commit 2e41297bfa035fa1cf1530d5ceecf58c527b6277 upstream. TDX host core code implements three seamcall*() helpers to make SEAMCALLs to the TDX module. Currently, they are implemented in and are exposed to other kernel code which includes . However, other than the TDX host core, seamcall*() are not expected to be used by other kernel code directly. For instance, for all SEAMCALLs that are used by KVM, the TDX host core exports a wrapper function for each of them. Move seamcall*() and related code out of and make them only visible to TDX host core. Since TDX host core tdx.c is already very heavy, don't put low level seamcall*() code there but to a new dedicated "seamcall_internal.h". Also, currently tdx.c has seamcall_prerr*() helpers which additionally print error message when calling seamcall*() fails. Move them to "seamcall_internal.h" as well. In such way all low level SEAMCALL helpers are in a dedicated place, which is much more readable. Copy the copyright notice from the original files and consolidate the date ranges to: Copyright (C) 2021-2023 Intel Corporation backport: drop: __seamcall_dirty_cache() sc_retry() without __seamcall_dirty_cache() Intel-SIG: commit 2e41297bfa03 x86/virt/tdx: Move low level SEAMCALL helpers out of Signed-off-by: Kai Huang Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Zhenzhong Duan Reviewed-by: Binbin Wu Reviewed-by: Tony Lindgren Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Xiaoyao Li Reviewed-by: Vishal Annapurve Acked-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-6-chao.gao@intel.com Link: https://patch.msgid.link/20260520222851.AF60554C@davehans-spike.ostc.intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx.h | 23 ------ arch/x86/virt/vmx/tdx/seamcall_internal.h | 88 +++++++++++++++++++++++ arch/x86/virt/vmx/tdx/tdx.c | 47 +----------- 3 files changed, 90 insertions(+), 68 deletions(-) create mode 100644 arch/x86/virt/vmx/tdx/seamcall_internal.h diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index c278263985b1..75dbb3c9513a 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -100,31 +100,8 @@ static inline long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, #endif /* CONFIG_INTEL_TDX_GUEST && CONFIG_KVM_GUEST */ #ifdef CONFIG_INTEL_TDX_HOST -u64 __seamcall(u64 fn, struct tdx_module_args *args); -u64 __seamcall_ret(u64 fn, struct tdx_module_args *args); -u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args); void tdx_init(void); -#include - -typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args); - -static __always_inline u64 sc_retry(sc_func_t func, u64 fn, - struct tdx_module_args *args) -{ - int retry = RDRAND_RETRY_LOOPS; - u64 ret; - - do { - ret = func(fn, args); - } while (ret == TDX_RND_NO_ENTROPY && --retry); - - return ret; -} - -#define seamcall(_fn, _args) sc_retry(__seamcall, (_fn), (_args)) -#define seamcall_ret(_fn, _args) sc_retry(__seamcall_ret, (_fn), (_args)) -#define seamcall_saved_ret(_fn, _args) sc_retry(__seamcall_saved_ret, (_fn), (_args)) const char *tdx_dump_mce_info(struct mce *m); const struct tdx_sys_info *tdx_get_sysinfo(void); diff --git a/arch/x86/virt/vmx/tdx/seamcall_internal.h b/arch/x86/virt/vmx/tdx/seamcall_internal.h new file mode 100644 index 000000000000..bc9fc2b113e5 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/seamcall_internal.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * SEAMCALL utilities for TDX host-side operations. + * + * Provides convenient wrappers around SEAMCALL assembly with retry logic, + * error reporting and cache coherency tracking. + * + * Copyright (C) 2021-2023 Intel Corporation + */ + +#ifndef _X86_VIRT_SEAMCALL_INTERNAL_H +#define _X86_VIRT_SEAMCALL_INTERNAL_H + +#include +#include +#include +#include +#include + +u64 __seamcall(u64 fn, struct tdx_module_args *args); +u64 __seamcall_ret(u64 fn, struct tdx_module_args *args); +u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args); + +typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args); + +static __always_inline u64 sc_retry(sc_func_t func, u64 fn, + struct tdx_module_args *args) +{ + int retry = RDRAND_RETRY_LOOPS; + u64 ret; + + do { + ret = func(fn, args); + } while (ret == TDX_RND_NO_ENTROPY && --retry); + + return ret; +} + +#define seamcall(_fn, _args) sc_retry(__seamcall, (_fn), (_args)) +#define seamcall_ret(_fn, _args) sc_retry(__seamcall_ret, (_fn), (_args)) +#define seamcall_saved_ret(_fn, _args) sc_retry(__seamcall_saved_ret, (_fn), (_args)) + +typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); + +static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args) +{ + pr_err("SEAMCALL (0x%016llx) failed: 0x%016llx\n", fn, err); +} + +static inline void seamcall_err_ret(u64 fn, u64 err, + struct tdx_module_args *args) +{ + seamcall_err(fn, err, args); + pr_err("RCX 0x%016llx RDX 0x%016llx R08 0x%016llx\n", + args->rcx, args->rdx, args->r8); + pr_err("R09 0x%016llx R10 0x%016llx R11 0x%016llx\n", + args->r9, args->r10, args->r11); +} + +static __always_inline int sc_retry_prerr(sc_func_t func, + sc_err_func_t err_func, + u64 fn, struct tdx_module_args *args) +{ + u64 sret = sc_retry(func, fn, args); + + if (sret == TDX_SUCCESS) + return 0; + + if (sret == TDX_SEAMCALL_VMFAILINVALID) + return -ENODEV; + + if (sret == TDX_SEAMCALL_GP) + return -EOPNOTSUPP; + + if (sret == TDX_SEAMCALL_UD) + return -EACCES; + + err_func(fn, sret, args); + return -EIO; +} + +#define seamcall_prerr(__fn, __args) \ + sc_retry_prerr(__seamcall, seamcall_err, (__fn), (__args)) + +#define seamcall_prerr_ret(__fn, __args) \ + sc_retry_prerr(__seamcall_ret, seamcall_err_ret, (__fn), (__args)) + +#endif /* _X86_VIRT_SEAMCALL_INTERNAL_H */ diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 90930f5fd311..d593a9f2ec49 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -41,6 +41,8 @@ #include #include #include + +#include "seamcall_internal.h" #include "tdx.h" struct tdx_module_state { @@ -65,51 +67,6 @@ static LIST_HEAD(tdx_memlist); static struct tdx_sys_info tdx_sysinfo __ro_after_init; -typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); - -static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args) -{ - pr_err("SEAMCALL (0x%016llx) failed: 0x%016llx\n", fn, err); -} - -static inline void seamcall_err_ret(u64 fn, u64 err, - struct tdx_module_args *args) -{ - seamcall_err(fn, err, args); - pr_err("RCX 0x%016llx RDX 0x%016llx R08 0x%016llx\n", - args->rcx, args->rdx, args->r8); - pr_err("R09 0x%016llx R10 0x%016llx R11 0x%016llx\n", - args->r9, args->r10, args->r11); -} - -static __always_inline int sc_retry_prerr(sc_func_t func, - sc_err_func_t err_func, - u64 fn, struct tdx_module_args *args) -{ - u64 sret = sc_retry(func, fn, args); - - if (sret == TDX_SUCCESS) - return 0; - - if (sret == TDX_SEAMCALL_VMFAILINVALID) - return -ENODEV; - - if (sret == TDX_SEAMCALL_GP) - return -EOPNOTSUPP; - - if (sret == TDX_SEAMCALL_UD) - return -EACCES; - - err_func(fn, sret, args); - return -EIO; -} - -#define seamcall_prerr(__fn, __args) \ - sc_retry_prerr(__seamcall, seamcall_err, (__fn), (__args)) - -#define seamcall_prerr_ret(__fn, __args) \ - sc_retry_prerr(__seamcall_ret, seamcall_err_ret, (__fn), (__args)) - static DEFINE_RAW_SPINLOCK(sysinit_lock); /* -- Gitee From f57bbfb692d49e9bef85a45b5b782b435835e25b Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 10 Feb 2025 13:30:25 +0100 Subject: [PATCH 45/78] driver core: add a faux bus for use when a simple device/bus is needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ANBZ: #40036 commit 35fa2d88ca9481e5caf533d58b99ca259c63b2fe upstream. Many drivers abuse the platform driver/bus system as it provides a simple way to create and bind a device to a driver-specific set of probe/release functions. Instead of doing that, and wasting all of the memory associated with a platform device, here is a "faux" bus that can be used instead. backprot: Without this patch: driver core: have match() callback in struct bus_type take a const * static int faux_match(struct device *dev, const struct device_driver *drv) --> static int faux_match(struct device *dev, struct device_driver *drv) Intel-SIG: commit 35fa2d88ca94 driver core: add a faux bus for use when a simple device/bus is needed Reviewed-by: Jonathan Cameron Reviewed-by: Danilo Krummrich Reviewed-by: Lyude Paul Reviewed-by: Thomas Weißschuh Reviewed-by: Zijun Hu Link: https://lore.kernel.org/r/2025021026-atlantic-gibberish-3f0c@gregkh Signed-off-by: Greg Kroah-Hartman [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/driver-api/infrastructure.rst | 6 + drivers/base/Makefile | 2 +- drivers/base/base.h | 1 + drivers/base/faux.c | 232 ++++++++++++++++++++ drivers/base/init.c | 1 + include/linux/device/faux.h | 69 ++++++ 6 files changed, 310 insertions(+), 1 deletion(-) create mode 100644 drivers/base/faux.c create mode 100644 include/linux/device/faux.h diff --git a/Documentation/driver-api/infrastructure.rst b/Documentation/driver-api/infrastructure.rst index 3d52dfdfa9fd..35e36fee4238 100644 --- a/Documentation/driver-api/infrastructure.rst +++ b/Documentation/driver-api/infrastructure.rst @@ -41,6 +41,12 @@ Device Drivers Base .. kernel-doc:: drivers/base/class.c :export: +.. kernel-doc:: include/linux/device/faux.h + :internal: + +.. kernel-doc:: drivers/base/faux.c + :export: + .. kernel-doc:: drivers/base/node.c :internal: diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 3079bfe53d04..0c6d88394a75 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -6,7 +6,7 @@ obj-y := component.o core.o bus.o dd.o syscore.o \ cpu.o firmware.o init.o map.o devres.o \ attribute_container.o transport_class.o \ topology.o container.o property.o cacheinfo.o \ - swnode.o + swnode.o faux.o obj-$(CONFIG_AUXILIARY_BUS) += auxiliary.o obj-$(CONFIG_DEVTMPFS) += devtmpfs.o obj-y += power/ diff --git a/drivers/base/base.h b/drivers/base/base.h index 0b491449b022..bfbec995a8d4 100644 --- a/drivers/base/base.h +++ b/drivers/base/base.h @@ -138,6 +138,7 @@ int hypervisor_init(void); static inline int hypervisor_init(void) { return 0; } #endif int platform_bus_init(void); +int faux_bus_init(void); void cpu_dev_init(void); void container_dev_init(void); #ifdef CONFIG_AUXILIARY_BUS diff --git a/drivers/base/faux.c b/drivers/base/faux.c new file mode 100644 index 000000000000..7d4f61437606 --- /dev/null +++ b/drivers/base/faux.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Greg Kroah-Hartman + * Copyright (c) 2025 The Linux Foundation + * + * A "simple" faux bus that allows devices to be created and added + * automatically to it. This is to be used whenever you need to create a + * device that is not associated with any "real" system resources, and do + * not want to have to deal with a bus/driver binding logic. It is + * intended to be very simple, with only a create and a destroy function + * available. + */ +#include +#include +#include +#include +#include +#include +#include "base.h" + +/* + * Internal wrapper structure so we can hold a pointer to the + * faux_device_ops for this device. + */ +struct faux_object { + struct faux_device faux_dev; + const struct faux_device_ops *faux_ops; +}; +#define to_faux_object(dev) container_of_const(dev, struct faux_object, faux_dev.dev) + +static struct device faux_bus_root = { + .init_name = "faux", +}; + +static int faux_match(struct device *dev, struct device_driver *drv) +{ + /* Match always succeeds, we only have one driver */ + return 1; +} + +static int faux_probe(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + struct faux_device *faux_dev = &faux_obj->faux_dev; + const struct faux_device_ops *faux_ops = faux_obj->faux_ops; + int ret = 0; + + if (faux_ops && faux_ops->probe) + ret = faux_ops->probe(faux_dev); + + return ret; +} + +static void faux_remove(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + struct faux_device *faux_dev = &faux_obj->faux_dev; + const struct faux_device_ops *faux_ops = faux_obj->faux_ops; + + if (faux_ops && faux_ops->remove) + faux_ops->remove(faux_dev); +} + +static const struct bus_type faux_bus_type = { + .name = "faux", + .match = faux_match, + .probe = faux_probe, + .remove = faux_remove, +}; + +static struct device_driver faux_driver = { + .name = "faux_driver", + .bus = &faux_bus_type, + .probe_type = PROBE_FORCE_SYNCHRONOUS, +}; + +static void faux_device_release(struct device *dev) +{ + struct faux_object *faux_obj = to_faux_object(dev); + + kfree(faux_obj); +} + +/** + * faux_device_create_with_groups - Create and register with the driver + * core a faux device and populate the device with an initial + * set of sysfs attributes. + * @name: The name of the device we are adding, must be unique for + * all faux devices. + * @parent: Pointer to a potential parent struct device. If set to + * NULL, the device will be created in the "root" of the faux + * device tree in sysfs. + * @faux_ops: struct faux_device_ops that the new device will call back + * into, can be NULL. + * @groups: The set of sysfs attributes that will be created for this + * device when it is registered with the driver core. + * + * Create a new faux device and register it in the driver core properly. + * If present, callbacks in @faux_ops will be called with the device that + * for the caller to do something with at the proper time given the + * device's lifecycle. + * + * Note, when this function is called, the functions specified in struct + * faux_ops can be called before the function returns, so be prepared for + * everything to be properly initialized before that point in time. + * + * Return: + * * NULL if an error happened with creating the device + * * pointer to a valid struct faux_device that is registered with sysfs + */ +struct faux_device *faux_device_create_with_groups(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops, + const struct attribute_group **groups) +{ + struct faux_object *faux_obj; + struct faux_device *faux_dev; + struct device *dev; + int ret; + + faux_obj = kzalloc(sizeof(*faux_obj), GFP_KERNEL); + if (!faux_obj) + return NULL; + + /* Save off the callbacks so we can use them in the future */ + faux_obj->faux_ops = faux_ops; + + /* Initialize the device portion and register it with the driver core */ + faux_dev = &faux_obj->faux_dev; + dev = &faux_dev->dev; + + device_initialize(dev); + dev->release = faux_device_release; + if (parent) + dev->parent = parent; + else + dev->parent = &faux_bus_root; + dev->bus = &faux_bus_type; + dev->groups = groups; + dev_set_name(dev, "%s", name); + + ret = device_add(dev); + if (ret) { + pr_err("%s: device_add for faux device '%s' failed with %d\n", + __func__, name, ret); + put_device(dev); + return NULL; + } + + return faux_dev; +} +EXPORT_SYMBOL_GPL(faux_device_create_with_groups); + +/** + * faux_device_create - create and register with the driver core a faux device + * @name: The name of the device we are adding, must be unique for all + * faux devices. + * @parent: Pointer to a potential parent struct device. If set to + * NULL, the device will be created in the "root" of the faux + * device tree in sysfs. + * @faux_ops: struct faux_device_ops that the new device will call back + * into, can be NULL. + * + * Create a new faux device and register it in the driver core properly. + * If present, callbacks in @faux_ops will be called with the device that + * for the caller to do something with at the proper time given the + * device's lifecycle. + * + * Note, when this function is called, the functions specified in struct + * faux_ops can be called before the function returns, so be prepared for + * everything to be properly initialized before that point in time. + * + * Return: + * * NULL if an error happened with creating the device + * * pointer to a valid struct faux_device that is registered with sysfs + */ +struct faux_device *faux_device_create(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops) +{ + return faux_device_create_with_groups(name, parent, faux_ops, NULL); +} +EXPORT_SYMBOL_GPL(faux_device_create); + +/** + * faux_device_destroy - destroy a faux device + * @faux_dev: faux device to destroy + * + * Unregisters and cleans up a device that was created with a call to + * faux_device_create() + */ +void faux_device_destroy(struct faux_device *faux_dev) +{ + struct device *dev = &faux_dev->dev; + + if (!faux_dev) + return; + + device_del(dev); + + /* The final put_device() will clean up the memory we allocated for this device. */ + put_device(dev); +} +EXPORT_SYMBOL_GPL(faux_device_destroy); + +int __init faux_bus_init(void) +{ + int ret; + + ret = device_register(&faux_bus_root); + if (ret) { + put_device(&faux_bus_root); + return ret; + } + + ret = bus_register(&faux_bus_type); + if (ret) + goto error_bus; + + ret = driver_register(&faux_driver); + if (ret) + goto error_driver; + + return ret; + +error_driver: + bus_unregister(&faux_bus_type); + +error_bus: + device_unregister(&faux_bus_root); + return ret; +} diff --git a/drivers/base/init.c b/drivers/base/init.c index 397eb9880cec..c9c04b664fda 100644 --- a/drivers/base/init.c +++ b/drivers/base/init.c @@ -32,6 +32,7 @@ void __init driver_init(void) /* These are also core pieces, but must come after the * core core pieces. */ + faux_bus_init(); of_core_init(); platform_bus_init(); auxiliary_bus_init(); diff --git a/include/linux/device/faux.h b/include/linux/device/faux.h new file mode 100644 index 000000000000..9f43c0e46aa4 --- /dev/null +++ b/include/linux/device/faux.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025 Greg Kroah-Hartman + * Copyright (c) 2025 The Linux Foundation + * + * A "simple" faux bus that allows devices to be created and added + * automatically to it. This is to be used whenever you need to create a + * device that is not associated with any "real" system resources, and do + * not want to have to deal with a bus/driver binding logic. It is + * intended to be very simple, with only a create and a destroy function + * available. + */ +#ifndef _FAUX_DEVICE_H_ +#define _FAUX_DEVICE_H_ + +#include +#include + +/** + * struct faux_device - a "faux" device + * @dev: internal struct device of the object + * + * A simple faux device that can be created/destroyed. To be used when a + * driver only needs to have a device to "hang" something off. This can be + * used for downloading firmware or other basic tasks. Use this instead of + * a struct platform_device if the device has no resources assigned to + * it at all. + */ +struct faux_device { + struct device dev; +}; +#define to_faux_device(x) container_of_const((x), struct faux_device, dev) + +/** + * struct faux_device_ops - a set of callbacks for a struct faux_device + * @probe: called when a faux device is probed by the driver core + * before the device is fully bound to the internal faux bus + * code. If probe succeeds, return 0, otherwise return a + * negative error number to stop the probe sequence from + * succeeding. + * @remove: called when a faux device is removed from the system + * + * Both @probe and @remove are optional, if not needed, set to NULL. + */ +struct faux_device_ops { + int (*probe)(struct faux_device *faux_dev); + void (*remove)(struct faux_device *faux_dev); +}; + +struct faux_device *faux_device_create(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops); +struct faux_device *faux_device_create_with_groups(const char *name, + struct device *parent, + const struct faux_device_ops *faux_ops, + const struct attribute_group **groups); +void faux_device_destroy(struct faux_device *faux_dev); + +static inline void *faux_device_get_drvdata(const struct faux_device *faux_dev) +{ + return dev_get_drvdata(&faux_dev->dev); +} + +static inline void faux_device_set_drvdata(struct faux_device *faux_dev, void *data) +{ + dev_set_drvdata(&faux_dev->dev, data); +} + +#endif /* _FAUX_DEVICE_H_ */ -- Gitee From bf2e2f4b42b1b7acb764bda6a43fc8677df9195a Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:52 -0700 Subject: [PATCH 46/78] coco/tdx-host: Introduce a "tdx_host" device ANBZ: #40036 commit 59783353467958517a3e511394d7ab3aed03bc6a upstream. TDX depends on a platform firmware module that runs on the CPU. Unlike other CoCo architectures, TDX has no hardware "device" running the show, just a blob on the CPU. Create a virtual device to anchor interactions with this platform firmware. This lets later code: - expose metadata: TDX module version, seamldr version, to userspace as device attributes - implement firmware uploader APIs (which are tied to a device) to support TDX module runtime updates Use a faux device because the TDX module is singular within the system and has no platform resources. Using a faux device eliminates the need to create a stub bus. The call to tdx_get_sysinfo() ensures that the TDX module is ready to provide services. Note that AMD has a PCI device for the PSP for SEV and ARM CCA will likely have a faux device [1]. Thanks to Dan and Yilun for all the help on this one. Intel-SIG: commit 597833534679 coco/tdx-host: Introduce a "tdx_host" device [ dhansen: trim changelog ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Jonathan Cameron Reviewed-by: Tony Lindgren Reviewed-by: Xu Yilun Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Xiaoyao Li Link: https://lore.kernel.org/all/2025073035-bulginess-rematch-b92e@gregkh/ # [1] Link: https://patch.msgid.link/20260520133909.409394-7-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 2 +- drivers/virt/coco/Kconfig | 2 ++ drivers/virt/coco/Makefile | 1 + drivers/virt/coco/tdx-host/Kconfig | 4 +++ drivers/virt/coco/tdx-host/Makefile | 1 + drivers/virt/coco/tdx-host/tdx-host.c | 43 +++++++++++++++++++++++++++ 6 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 drivers/virt/coco/tdx-host/Kconfig create mode 100644 drivers/virt/coco/tdx-host/Makefile create mode 100644 drivers/virt/coco/tdx-host/tdx-host.c diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index d593a9f2ec49..31d3c919be09 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1494,7 +1494,7 @@ const struct tdx_sys_info *tdx_get_sysinfo(void) return (const struct tdx_sys_info *)&tdx_sysinfo; } -EXPORT_SYMBOL_FOR_KVM(tdx_get_sysinfo); +EXPORT_SYMBOL_FOR_MODULES(tdx_get_sysinfo, "kvm-intel,tdx-host"); u32 tdx_get_nr_guest_keyids(void) { diff --git a/drivers/virt/coco/Kconfig b/drivers/virt/coco/Kconfig index 0f50064762c2..5e29b118df4d 100644 --- a/drivers/virt/coco/Kconfig +++ b/drivers/virt/coco/Kconfig @@ -19,3 +19,5 @@ source "drivers/virt/coco/sev-guest/Kconfig" source "drivers/virt/coco/tdx-guest/Kconfig" source "drivers/virt/coco/csv-guest/Kconfig" + +source "drivers/virt/coco/tdx-host/Kconfig" diff --git a/drivers/virt/coco/Makefile b/drivers/virt/coco/Makefile index 56583de55c08..afb6653a0c70 100644 --- a/drivers/virt/coco/Makefile +++ b/drivers/virt/coco/Makefile @@ -7,4 +7,5 @@ obj-$(CONFIG_TSM_MEASUREMENTS) += tsm-mr.o obj-$(CONFIG_EFI_SECRET) += efi_secret/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ +obj-$(CONFIG_INTEL_TDX_HOST) += tdx-host/ obj-$(CONFIG_CSV_GUEST) += csv-guest/ diff --git a/drivers/virt/coco/tdx-host/Kconfig b/drivers/virt/coco/tdx-host/Kconfig new file mode 100644 index 000000000000..cfe81b9c0364 --- /dev/null +++ b/drivers/virt/coco/tdx-host/Kconfig @@ -0,0 +1,4 @@ +config TDX_HOST_SERVICES + tristate + depends on INTEL_TDX_HOST + default m diff --git a/drivers/virt/coco/tdx-host/Makefile b/drivers/virt/coco/tdx-host/Makefile new file mode 100644 index 000000000000..e61e749a8dff --- /dev/null +++ b/drivers/virt/coco/tdx-host/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_TDX_HOST_SERVICES) += tdx-host.o diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c new file mode 100644 index 000000000000..c77885392b09 --- /dev/null +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TDX host user interface driver + * + * Copyright (C) 2025 Intel Corporation + */ + +#include +#include +#include + +#include +#include + +static const struct x86_cpu_id tdx_host_ids[] = { + X86_MATCH_FEATURE(X86_FEATURE_TDX_HOST_PLATFORM, NULL), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, tdx_host_ids); + +static struct faux_device *fdev; + +static int __init tdx_host_init(void) +{ + if (!x86_match_cpu(tdx_host_ids) || !tdx_get_sysinfo()) + return -ENODEV; + + fdev = faux_device_create(KBUILD_MODNAME, NULL, NULL); + if (!fdev) + return -ENODEV; + + return 0; +} +module_init(tdx_host_init); + +static void __exit tdx_host_exit(void) +{ + faux_device_destroy(fdev); +} +module_exit(tdx_host_exit); + +MODULE_DESCRIPTION("TDX Host Services"); +MODULE_LICENSE("GPL"); -- Gitee From 30403ee8aeee81f3d88f9df6b53da00812ba9fb7 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:53 -0700 Subject: [PATCH 47/78] coco/tdx-host: Expose TDX module version ANBZ: #40036 commit 6fe8a33fdb93911934cd1c77624c1e02af45047c upstream. For TDX module updates, userspace needs to select compatible update versions based on the current module version. For example, the 1.5.x series runs on Sapphire Rapids but not Granite Rapids, which needs 2.0.x. Updates are also constrained by version distance, so a 1.5.6 module might permit updates to 1.5.7 but not to 1.5.20. Start the process of punting the version selection logic to userspace. Expose the TDX module version in the new faux device. Define TDX_VERSION_FMT macro for the TDX version format since it will be used multiple times. Also convert an existing print statement to use it. == Background == For posterity, here's what other firmware mechanisms do: 1. AMD SEV leverages an existing PCI device for the PSP to expose metadata. TDX uses a faux device as it doesn't have PCI device in its architecture. 2. Microcode uses per-CPU virtual devices to report microcode revisions because CPUs can have different revisions. But, there is only a single TDX module, so exposing the TDX module version through a global TDX faux device is appropriate 3. ARM's CCA implementation isn't in-tree yet, but will likely follow a similar faux device approach, though it's unclear whether they need to expose firmware version information Intel-SIG: commit 6fe8a33fdb93 coco/tdx-host: Expose TDX module version [ dhansen: trim changelog ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Binbin Wu Reviewed-by: Tony Lindgren Reviewed-by: Xu Yilun Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Xiaoyao Li Reviewed-by: Dave Hansen Link: https://lore.kernel.org/all/2025073035-bulginess-rematch-b92e@gregkh/ # [1] Link: https://patch.msgid.link/20260520133909.409394-8-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- .../ABI/testing/sysfs-devices-faux-tdx-host | 5 ++++ arch/x86/include/asm/tdx.h | 6 +++++ arch/x86/virt/vmx/tdx/tdx_global_metadata.c | 2 +- drivers/virt/coco/tdx-host/tdx-host.c | 26 ++++++++++++++++++- 4 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-devices-faux-tdx-host diff --git a/Documentation/ABI/testing/sysfs-devices-faux-tdx-host b/Documentation/ABI/testing/sysfs-devices-faux-tdx-host new file mode 100644 index 000000000000..47d73cb89f1e --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-faux-tdx-host @@ -0,0 +1,5 @@ +What: /sys/devices/faux/tdx_host/version +Contact: linux-coco@lists.linux.dev +Description: (RO) Report the version of the loaded TDX module. + Formatted as "major.minor.update". Used by TDX module + update tooling. Example: "1.2.03". diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 75dbb3c9513a..df32ef57833e 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -41,6 +41,12 @@ #include #include +/* + * TDX module and P-SEAMLDR version convention: "major.minor.update" + * (e.g., "1.5.08") with zero-padded two-digit update field. + */ +#define TDX_VERSION_FMT "%u.%u.%02u" + /* * Used by the #VE exception handler to gather the #VE exception * info from the TDX module. This is a software only structure diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c index e854aa0ca649..982b7471e517 100644 --- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -106,7 +106,7 @@ static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo) ret = ret ?: get_tdx_sys_info_version(&sysinfo->version); - pr_info("Module version: %u.%u.%02u\n", + pr_info("Module version: " TDX_VERSION_FMT "\n", sysinfo->version.major_version, sysinfo->version.minor_version, sysinfo->version.update_version); diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c index c77885392b09..ef117a836b3a 100644 --- a/drivers/virt/coco/tdx-host/tdx-host.c +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -18,6 +19,29 @@ static const struct x86_cpu_id tdx_host_ids[] = { }; MODULE_DEVICE_TABLE(x86cpu, tdx_host_ids); +static ssize_t version_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const struct tdx_sys_info *tdx_sysinfo = tdx_get_sysinfo(); + const struct tdx_sys_info_version *ver; + + if (!tdx_sysinfo) + return -ENXIO; + + ver = &tdx_sysinfo->version; + + return sysfs_emit(buf, TDX_VERSION_FMT "\n", ver->major_version, + ver->minor_version, + ver->update_version); +} +static DEVICE_ATTR_RO(version); + +static struct attribute *tdx_host_attrs[] = { + &dev_attr_version.attr, + NULL, +}; +ATTRIBUTE_GROUPS(tdx_host); + static struct faux_device *fdev; static int __init tdx_host_init(void) @@ -25,7 +49,7 @@ static int __init tdx_host_init(void) if (!x86_match_cpu(tdx_host_ids) || !tdx_get_sysinfo()) return -ENODEV; - fdev = faux_device_create(KBUILD_MODNAME, NULL, NULL); + fdev = faux_device_create_with_groups(KBUILD_MODNAME, NULL, NULL, tdx_host_groups); if (!fdev) return -ENODEV; -- Gitee From 4ebaf5bc926a0ebe330fed16693f6ae2fd711a0a Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:55 -0700 Subject: [PATCH 48/78] x86/virt/seamldr: Introduce a wrapper for P-SEAMLDR SEAMCALLs ANBZ: #40036 commit 9bc3ce2c5d3ddcbf569e573c1c65775c09a4fb75 upstream. The TDX architecture uses the "SEAMCALL" instruction to communicate with SEAM mode software. Right now, the only SEAM mode software that the kernel communicates with is the TDX module. But, there is actually another component that runs in SEAM mode but it is separate from the TDX module: the persistent SEAM loader or "P-SEAMLDR". Right now, the only component that communicates with it is the BIOS which loads the TDX module itself at boot. But, to support updating the TDX module, the kernel now needs to be able to talk to it. P-SEAMLDR SEAMCALLs differ from TDX module SEAMCALLs in areas such as concurrency requirements. Add a P-SEAMLDR wrapper to handle these differences and prepare for implementing concrete functions. Use seamcall_prerr() (not '_ret') because current P-SEAMLDR calls do not use any output registers other than RAX. Note: Despite the similar name, the NP-SEAMLDR ("Non-Persistent") (ACM) invoked exclusively by the BIOS at boot rather than a component running in SEAM mode. The kernel cannot call it at runtime. It exposes no SEAMCALL interface. Intel-SIG: commit 9bc3ce2c5d3d x86/virt/seamldr: Introduce a wrapper for P-SEAMLDR SEAMCALLs Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Binbin Wu Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Dave Hansen Link: https://cdrdv2.intel.com/v1/dl/getContent/733582 # [1] Link: https://patch.msgid.link/20260520133909.409394-9-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/Makefile | 2 +- arch/x86/virt/vmx/tdx/seamldr.c | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 arch/x86/virt/vmx/tdx/seamldr.c diff --git a/arch/x86/virt/vmx/tdx/Makefile b/arch/x86/virt/vmx/tdx/Makefile index 90da47eb85ee..d1dbc5cc5697 100644 --- a/arch/x86/virt/vmx/tdx/Makefile +++ b/arch/x86/virt/vmx/tdx/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-y += seamcall.o tdx.o +obj-y += seamcall.o seamldr.o tdx.o diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c new file mode 100644 index 000000000000..65616dd2f4d2 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * P-SEAMLDR support for TDX module management features like runtime updates + * + * Copyright (C) 2025 Intel Corporation + */ +#define pr_fmt(fmt) "seamldr: " fmt + +#include + +#include "seamcall_internal.h" + +/* + * Serialize P-SEAMLDR calls since the hardware only allows a single CPU to + * interact with P-SEAMLDR simultaneously. Use raw version as the calls can + * be made with interrupts disabled, where plain spinlocks are prohibited in + * PREEMPT_RT kernels as they become sleeping locks. + */ +static DEFINE_RAW_SPINLOCK(seamldr_lock); + +static __maybe_unused int seamldr_call(u64 fn, struct tdx_module_args *args) +{ + guard(raw_spinlock)(&seamldr_lock); + return seamcall_prerr(fn, args); +} -- Gitee From 6e43a099439afea92e8d8b908302ba69f89db9f4 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:56 -0700 Subject: [PATCH 49/78] x86/virt/seamldr: Add a helper to retrieve P-SEAMLDR information ANBZ: #40036 commit 0988bf698a80056638f771179158059c60874ab4 upstream. P-SEAMLDR reports its state via SEAMLDR.INFO, including its version and the number of remaining runtime updates. This information is useful for userspace. For example, userspace can use the P-SEAMLDR version to determine whether a candidate TDX module is compatible with the running loader, and can use the remaining update count to determine whether another runtime update is still possible. Add a helper to retrieve P-SEAMLDR information in preparation for exposing P-SEAMLDR version and other necessary information to userspace. Export the new kAPI for use by the "tdx_host" device. Note that there are two distinct P-SEAMLDR APIs with similar names: "SEAMLDR.INFO" is metadata about the loader. It's metadata for the update process. "SEAMLDR.SEAMINFO" is metadata about SEAM mode. It is for the module init process, not for the update process. Use SEAMLDR.INFO here. For details, see "Intel Trust Domain Extensions - SEAM Loader (SEAMLDR) Interface Specification". Intel-SIG: commit 0988bf698a80 x86/virt/seamldr: Add a helper to retrieve P-SEAMLDR information Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Xiaoyao Li Reviewed-by: Rick Edgecombe Reviewed-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-10-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/seamldr.h | 35 +++++++++++++++++++++++++++++++++ arch/x86/virt/vmx/tdx/seamldr.c | 20 ++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 arch/x86/include/asm/seamldr.h diff --git a/arch/x86/include/asm/seamldr.h b/arch/x86/include/asm/seamldr.h new file mode 100644 index 000000000000..a74151b75599 --- /dev/null +++ b/arch/x86/include/asm/seamldr.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_SEAMLDR_H +#define _ASM_X86_SEAMLDR_H + +#include + +/* + * This is the "SEAMLDR_INFO" data structure defined in the + * "SEAM Loader (SEAMLDR) Interface Specification". + * + * Must be aligned to a 256-byte boundary. + */ +struct seamldr_info { + u32 version; + u32 attributes; + u32 vendor_id; + u32 build_date; + u16 build_num; + u16 minor_version; + u16 major_version; + u16 update_version; + u32 acm_x2apicid; + u32 num_remaining_updates; + u8 seam_info[128]; + u8 seam_ready; + u8 seam_debug; + u8 p_seam_ready; + u8 reserved[93]; +} __packed __aligned(256); + +static_assert(sizeof(struct seamldr_info) == 256); + +int seamldr_get_info(struct seamldr_info *seamldr_info); + +#endif /* _ASM_X86_SEAMLDR_H */ diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 65616dd2f4d2..7269a239bc22 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -8,8 +8,13 @@ #include +#include + #include "seamcall_internal.h" +/* P-SEAMLDR SEAMCALL leaf function */ +#define P_SEAMLDR_INFO 0x8000000000000000 + /* * Serialize P-SEAMLDR calls since the hardware only allows a single CPU to * interact with P-SEAMLDR simultaneously. Use raw version as the calls can @@ -18,8 +23,21 @@ */ static DEFINE_RAW_SPINLOCK(seamldr_lock); -static __maybe_unused int seamldr_call(u64 fn, struct tdx_module_args *args) +static int seamldr_call(u64 fn, struct tdx_module_args *args) { guard(raw_spinlock)(&seamldr_lock); return seamcall_prerr(fn, args); } + +int seamldr_get_info(struct seamldr_info *seamldr_info) +{ + struct tdx_module_args args = {}; + + /* + * Use slow_virt_to_phys() since @seamldr_info may be allocated on + * the stack. + */ + args.rcx = slow_virt_to_phys(seamldr_info); + return seamldr_call(P_SEAMLDR_INFO, &args); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); -- Gitee From 6eff16ca47f3ea56ca81e707ca29f2d3140ab0a7 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:57 -0700 Subject: [PATCH 50/78] coco/tdx-host: Expose P-SEAMLDR information via sysfs ANBZ: #40036 commit 885afcf7c859729d0d7884d0abe5343ee8f742a3 upstream. TDX module updates require userspace to select the appropriate module to load. Expose necessary information to facilitate this decision. Two values are needed: - P-SEAMLDR version: for compatibility checks between TDX module and P-SEAMLDR - num_remaining_updates: indicates how many updates can be performed Expose them as tdx-host device attributes visible only when updates are supported. Note that the underlying P-SEAMLDR attributes are available regardless of update support; this only restricts their visibility to userspace. Intel-SIG: commit 885afcf7c859 coco/tdx-host: Expose P-SEAMLDR information via sysfs Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-11-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- .../ABI/testing/sysfs-devices-faux-tdx-host | 21 ++++++ arch/x86/include/asm/tdx.h | 6 ++ drivers/virt/coco/tdx-host/tdx-host.c | 72 ++++++++++++++++++- 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-devices-faux-tdx-host b/Documentation/ABI/testing/sysfs-devices-faux-tdx-host index 47d73cb89f1e..c9cb273abf32 100644 --- a/Documentation/ABI/testing/sysfs-devices-faux-tdx-host +++ b/Documentation/ABI/testing/sysfs-devices-faux-tdx-host @@ -3,3 +3,24 @@ Contact: linux-coco@lists.linux.dev Description: (RO) Report the version of the loaded TDX module. Formatted as "major.minor.update". Used by TDX module update tooling. Example: "1.2.03". + +What: /sys/devices/faux/tdx_host/seamldr_version +Contact: linux-coco@lists.linux.dev +Description: (RO) Report the version of the loaded P-SEAMLDR. + Formatted as a TDX module version. Used by TDX module + update tooling. + +What: /sys/devices/faux/tdx_host/num_remaining_updates +Contact: linux-coco@lists.linux.dev +Description: (RO) Report the number of remaining updates. TDX maintains a + log about each TDX module that has been loaded. This log has + a finite size, which limits the number of TDX module updates + that can be performed. + + After each successful update, the number reduces by one. Once it + reaches zero, further updates will fail until next reboot. The + number is always zero if the P-SEAMLDR doesn't support updates. + + See Intel Trust Domain Extensions - SEAM Loader (SEAMLDR) + Interface Specification, Chapter "SEAMLDR_INFO" and Chapter + "SEAMLDR.INSTALL" for more information. diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index df32ef57833e..f27fbd9b11aa 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -111,6 +111,12 @@ void tdx_init(void); const char *tdx_dump_mce_info(struct mce *m); const struct tdx_sys_info *tdx_get_sysinfo(void); +static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinfo) +{ + /* To be enabled when kernel is ready. */ + return false; +} + int tdx_guest_keyid_alloc(void); u32 tdx_get_nr_guest_keyids(void); void tdx_guest_keyid_free(unsigned int keyid); diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c index ef117a836b3a..2997311f72fa 100644 --- a/drivers/virt/coco/tdx-host/tdx-host.c +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -11,6 +11,7 @@ #include #include +#include #include static const struct x86_cpu_id tdx_host_ids[] = { @@ -40,7 +41,76 @@ static struct attribute *tdx_host_attrs[] = { &dev_attr_version.attr, NULL, }; -ATTRIBUTE_GROUPS(tdx_host); + +static const struct attribute_group tdx_host_group = { + .attrs = tdx_host_attrs, +}; + +static ssize_t seamldr_version_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct seamldr_info info; + int ret; + + ret = seamldr_get_info(&info); + if (ret) + return ret; + + return sysfs_emit(buf, TDX_VERSION_FMT "\n", info.major_version, + info.minor_version, + info.update_version); +} + +static ssize_t num_remaining_updates_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct seamldr_info info; + int ret; + + ret = seamldr_get_info(&info); + if (ret) + return ret; + + return sysfs_emit(buf, "%u\n", info.num_remaining_updates); +} + +/* + * These attributes are intended for managing TDX module updates. Reading + * them issues a slow, serialized P-SEAMLDR query, so keep them admin-only. + */ +static DEVICE_ATTR_ADMIN_RO(seamldr_version); +static DEVICE_ATTR_ADMIN_RO(num_remaining_updates); + +static struct attribute *seamldr_attrs[] = { + &dev_attr_seamldr_version.attr, + &dev_attr_num_remaining_updates.attr, + NULL, +}; + +static umode_t seamldr_group_visible(struct kobject *kobj, struct attribute *attr, int idx) +{ + const struct tdx_sys_info *sysinfo = tdx_get_sysinfo(); + + if (!sysinfo) + return 0; + + if (!tdx_supports_runtime_update(sysinfo)) + return 0; + + return attr->mode; +} + +static const struct attribute_group seamldr_group = { + .attrs = seamldr_attrs, + .is_visible = seamldr_group_visible, +}; + +static const struct attribute_group *tdx_host_groups[] = { + &tdx_host_group, + &seamldr_group, + NULL, +}; static struct faux_device *fdev; -- Gitee From b90205ae535c382a3c074dc273c6734bed8288d4 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:28:59 -0700 Subject: [PATCH 51/78] coco/tdx-host: Don't expose P-SEAMLDR information on CPUs with erratum ANBZ: #40036 commit 5ce9cc5a232b992806cf31027e6281727a2040fc upstream. TDX-capable CPUs clobber the current VMCS on P-SEAMLDR calls. Clearing the current VMCS behind KVM's back breaks KVM. Future CPUs will fix this by preserving the current VMCS across P-SEAMLDR calls. A future specification update will describe the VMCS-clearing behavior as an erratum and to state that it does not occur when IA32_VMX_BASIC[60] is set. Add a CPU bug bit and refuse to expose P-SEAMLDR information on affected CPUs. Use a CPU bug bit to stay consistent with X86_BUG_TDX_PW_MCE. As a bonus, the bug bit is visible to userspace, which allows userspace to determine why these sysfs files are not exposed, and it can also be checked by other kernel components in the future if needed. == Alternatives == Two workarounds were considered but both were rejected: 1. Save/restore the current VMCS around P-SEAMLDR calls. This produces ugly assembly code [1] and doesn't play well with #MCE or #NMI if they need to use the current VMCS. 2. Move KVM's VMCS tracking logic to the TDX core code, which would break the boundary between KVM and the TDX core code [2]. Intel-SIG: commit 5ce9cc5a232b coco/tdx-host: Don't expose P-SEAMLDR information on CPUs with erratum [ dhansen: comment and changelog munging. Add seamldr_call() bug check. ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Reviewed-by: Dave Hansen Link: https://lore.kernel.org/kvm/fedb3192-e68c-423c-93b2-a4dc2f964148@intel.com/ # [1] Link: https://lore.kernel.org/kvm/aYIXFmT-676oN6j0@google.com/ # [2] Link: https://patch.msgid.link/20260520133909.409394-12-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/vmx.h | 1 + arch/x86/virt/vmx/tdx/seamldr.c | 13 +++++++++++++ arch/x86/virt/vmx/tdx/tdx.c | 11 +++++++++++ drivers/virt/coco/tdx-host/tdx-host.c | 8 ++++++++ 5 files changed, 34 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index a10632e049fc..4c143a5b22c7 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -590,4 +590,5 @@ #define X86_BUG_ITS_NATIVE_ONLY X86_BUG(1*32 + 6) /* CPU is affected by ITS, VMX is not affected */ #define X86_BUG_TSA X86_BUG(1*32+ 9) /* "tsa" CPU is affected by Transient Scheduler Attacks */ #define X86_BUG_VMSCAPE X86_BUG( 1*32+10) /* "vmscape" CPU is affected by VMSCAPE attacks from guests */ +#define X86_BUG_SEAMRET_INVD_VMCS X86_BUG( 1*32+11) /* "seamret_invd_vmcs" SEAMRET from P-SEAMLDR clears the current VMCS */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index acf6059d4811..8ce2c00d2b7c 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -151,6 +151,7 @@ struct vmcs { #define VMX_VMFUNC_EPTP_SWITCHING VMFUNC_CONTROL_BIT(EPTP_SWITCHING) #define VMFUNC_EPTP_ENTRIES 512 +#define VMX_BASIC_NO_SEAMRET_INVD_VMCS BIT_ULL(60) static inline u32 vmx_basic_vmcs_revision_id(u64 vmx_basic) { diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 7269a239bc22..baa86f2da04e 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -6,8 +6,11 @@ */ #define pr_fmt(fmt) "seamldr: " fmt +#include #include +#include +#include #include #include "seamcall_internal.h" @@ -25,6 +28,16 @@ static DEFINE_RAW_SPINLOCK(seamldr_lock); static int seamldr_call(u64 fn, struct tdx_module_args *args) { + /* + * With this bug, P-SEAMLDR calls corrupt the VMCS + * pointer and must be avoided. This path should be + * unreachable since sysfs hides the ABIs. + */ + if (boot_cpu_has_bug(X86_BUG_SEAMRET_INVD_VMCS)) { + WARN_ON(1); + return -EINVAL; + } + guard(raw_spinlock)(&seamldr_lock); return seamcall_prerr(fn, args); } diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 31d3c919be09..8b07b7c1011e 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "seamcall_internal.h" #include "tdx.h" @@ -1417,6 +1418,8 @@ static struct notifier_block tdx_memory_nb = { static void __init check_tdx_erratum(void) { + u64 basic_msr; + /* * These CPUs have an erratum. A partial write from non-TD * software (e.g. via MOVNTI variants or UC/WC mapping) to TDX @@ -1428,6 +1431,14 @@ static void __init check_tdx_erratum(void) case INTEL_FAM6_EMERALDRAPIDS_X: setup_force_cpu_bug(X86_BUG_TDX_PW_MCE); } + + /* + * Some TDX-capable CPUs have an erratum where the current VMCS is + * cleared after calling into P-SEAMLDR. + */ + rdmsrl(MSR_IA32_VMX_BASIC, basic_msr); + if (!(basic_msr & VMX_BASIC_NO_SEAMRET_INVD_VMCS)) + setup_force_cpu_bug(X86_BUG_SEAMRET_INVD_VMCS); } void __init tdx_init(void) diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c index 2997311f72fa..2886554f0e60 100644 --- a/drivers/virt/coco/tdx-host/tdx-host.c +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -98,6 +98,14 @@ static umode_t seamldr_group_visible(struct kobject *kobj, struct attribute *att if (!tdx_supports_runtime_update(sysinfo)) return 0; + /* + * This bug makes P-SEAMLDR calls clobber the current VMCS + * which breaks KVM. Avoid P-SEAMLDR calls by hiding all + * attributes if the CPU has this bug. + */ + if (boot_cpu_has_bug(X86_BUG_SEAMRET_INVD_VMCS)) + return 0; + return attr->mode; } -- Gitee From 6dfa76a6f6e59f790ebc7799f99e504b4568b218 Mon Sep 17 00:00:00 2001 From: Kory Maincent Date: Wed, 22 Nov 2023 14:52:43 +0100 Subject: [PATCH 52/78] firmware_loader: Expand Firmware upload error codes with firmware invalid error ANBZ: #40036 commit a066f906ba396ab00d4af19fc5fad42b2605582a upstream. No error code are available to signal an invalid firmware content. Drivers that can check the firmware content validity can not return this specific failure to the user-space Expand the firmware error code with an additional code: - "firmware invalid" code which can be used when the provided firmware is invalid Sync lib/test_firmware.c file accordingly. Intel-SIG: commit a066f906ba39 firmware_loader: Expand Firmware upload error codes with firmware invalid error Acked-by: Luis Chamberlain Acked-by: Greg Kroah-Hartman Signed-off-by: Kory Maincent Reviewed-by: Simon Horman Link: https://lore.kernel.org/r/20231122-feature_firmware_error_code-v3-1-04ec753afb71@bootlin.com Signed-off-by: Jakub Kicinski [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- drivers/base/firmware_loader/sysfs_upload.c | 1 + include/linux/firmware.h | 2 ++ lib/test_firmware.c | 1 + 3 files changed, 4 insertions(+) diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c index 54a1abae3948..cab79e7fcfcf 100644 --- a/drivers/base/firmware_loader/sysfs_upload.c +++ b/drivers/base/firmware_loader/sysfs_upload.c @@ -27,6 +27,7 @@ static const char * const fw_upload_err_str[] = { [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", + [FW_UPLOAD_ERR_FW_INVALID] = "firmware-invalid", }; static const char *fw_upload_progress(struct device *dev, diff --git a/include/linux/firmware.h b/include/linux/firmware.h index de7fea3bca51..0311858b46ce 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -27,6 +27,7 @@ struct firmware { * @FW_UPLOAD_ERR_INVALID_SIZE: invalid firmware image size * @FW_UPLOAD_ERR_RW_ERROR: read or write to HW failed, see kernel log * @FW_UPLOAD_ERR_WEAROUT: FLASH device is approaching wear-out, wait & retry + * @FW_UPLOAD_ERR_FW_INVALID: invalid firmware file * @FW_UPLOAD_ERR_MAX: Maximum error code marker */ enum fw_upload_err { @@ -38,6 +39,7 @@ enum fw_upload_err { FW_UPLOAD_ERR_INVALID_SIZE, FW_UPLOAD_ERR_RW_ERROR, FW_UPLOAD_ERR_WEAROUT, + FW_UPLOAD_ERR_FW_INVALID, FW_UPLOAD_ERR_MAX }; diff --git a/lib/test_firmware.c b/lib/test_firmware.c index add4699fc6cd..9cfdcd6d21db 100644 --- a/lib/test_firmware.c +++ b/lib/test_firmware.c @@ -1132,6 +1132,7 @@ static const char * const fw_upload_err_str[] = { [FW_UPLOAD_ERR_INVALID_SIZE] = "invalid-file-size", [FW_UPLOAD_ERR_RW_ERROR] = "read-write-error", [FW_UPLOAD_ERR_WEAROUT] = "flash-wearout", + [FW_UPLOAD_ERR_FW_INVALID] = "firmware-invalid", }; static void upload_err_inject_error(struct test_firmware_upload *tst, -- Gitee From da8c50a3632fa3e6d0dc3e8b2f73f9397f92f7ed Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:00 -0700 Subject: [PATCH 53/78] coco/tdx-host: Implement firmware upload sysfs ABI for TDX module updates ANBZ: #40036 commit c3e70c5ee53f1a5e1df2e83f846185154d58111f upstream. tl;dr: Select fw_upload for doing TDX module updates. The process of selecting among available update images is complicated and nuanced. Punt the selection process out to userspace. One existing userspace implementation today is the script in the Intel TDX Module Binaries repository[1]. Long Version: The kernel supports two primary firmware update mechanisms: 1. request_firmware() - used by microcode, SEV firmware, hundreds of other drivers 2. 'struct fw_upload' - used by CXL, FPGA updates, dozens of others The key difference between is that request_firmware() loads a named file from the filesystem where the filename is kernel-controlled, while fw_upload accepts firmware data directly from userspace. TDX module firmware update selection policy is too complex for the kernel. Leave it to userspace and use fw_upload. Add a skeleton fw_upload implementation to be fleshed out in subsequent patches. Refactor the sysfs visiblity attribute function so it can be used as a more generic flag for the presence of viable runtime update support. Why fw_upload instead of request_firmware()? ============================================ Selecting a TDX module update image is not a simple "load the latest" decision. Userspace needs to choose an image that is compatible with both the platform and the currently running module. Some constraints are hard requirements: a. Module version series are platform-specific. For example, the 1.5.x series runs on Sapphire Rapids but not Granite Rapids, which needs 2.0.x. b. Updates are also constrained by version distance. A 1.5.6 module might permit updates to 1.5.7 but not to 1.5.50. There may also be userspace policy choices: c. Decide the update direction: upgrade or downgrade d. Choose whether to optimize for fewer updates or smaller version steps, for example, 1.2.3=>1.2.5 versus 1.2.3=>1.2.4=>1.2.5. Given that complexity, leave module selection to userspace and use fw_upload. 1. https://github.com/intel/confidential-computing.tdx.tdx-module.binaries/blob/main/version_select_and_load.py [ dhansen: add version script link, add more explanation of code moves, fix some minor whitespace issues ] Intel-SIG: commit c3e70c5ee53f coco/tdx-host: Implement firmware upload sysfs ABI for TDX module updates Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Link: https://lore.kernel.org/kvm/01fc8946-eb84-46fa-9458-f345dd3f6033@intel.com/ Link: https://patch.msgid.link/20260520133909.409394-13-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/seamldr.h | 1 + arch/x86/virt/vmx/tdx/seamldr.c | 14 ++++ drivers/virt/coco/tdx-host/Kconfig | 2 + drivers/virt/coco/tdx-host/tdx-host.c | 93 +++++++++++++++++++++++++-- 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/seamldr.h b/arch/x86/include/asm/seamldr.h index a74151b75599..43084e2daa2d 100644 --- a/arch/x86/include/asm/seamldr.h +++ b/arch/x86/include/asm/seamldr.h @@ -31,5 +31,6 @@ struct seamldr_info { static_assert(sizeof(struct seamldr_info) == 256); int seamldr_get_info(struct seamldr_info *seamldr_info); +int seamldr_install_module(const u8 *data, u32 data_len); #endif /* _ASM_X86_SEAMLDR_H */ diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index baa86f2da04e..b2b6a439d417 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -54,3 +54,17 @@ int seamldr_get_info(struct seamldr_info *seamldr_info) return seamldr_call(P_SEAMLDR_INFO, &args); } EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); + +/** + * seamldr_install_module - Install a new TDX module. + * @data: Pointer to the TDX module image. + * @data_len: Size of the TDX module image. + * + * Returns 0 on success, negative error code on failure. + */ +int seamldr_install_module(const u8 *data, u32 data_len) +{ + /* TODO: Update TDX module here */ + return 0; +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_install_module, "tdx-host"); diff --git a/drivers/virt/coco/tdx-host/Kconfig b/drivers/virt/coco/tdx-host/Kconfig index cfe81b9c0364..57d0c01a4357 100644 --- a/drivers/virt/coco/tdx-host/Kconfig +++ b/drivers/virt/coco/tdx-host/Kconfig @@ -1,4 +1,6 @@ config TDX_HOST_SERVICES tristate depends on INTEL_TDX_HOST + select FW_LOADER + select FW_UPLOAD default m diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c index 2886554f0e60..e5a672be6200 100644 --- a/drivers/virt/coco/tdx-host/tdx-host.c +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -88,15 +89,15 @@ static struct attribute *seamldr_attrs[] = { NULL, }; -static umode_t seamldr_group_visible(struct kobject *kobj, struct attribute *attr, int idx) +static bool supports_runtime_update(void) { const struct tdx_sys_info *sysinfo = tdx_get_sysinfo(); if (!sysinfo) - return 0; + return false; if (!tdx_supports_runtime_update(sysinfo)) - return 0; + return false; /* * This bug makes P-SEAMLDR calls clobber the current VMCS @@ -104,6 +105,14 @@ static umode_t seamldr_group_visible(struct kobject *kobj, struct attribute *att * attributes if the CPU has this bug. */ if (boot_cpu_has_bug(X86_BUG_SEAMRET_INVD_VMCS)) + return false; + + return true; +} + +static umode_t seamldr_group_visible(struct kobject *kobj, struct attribute *attr, int idx) +{ + if (!supports_runtime_update()) return 0; return attr->mode; @@ -120,6 +129,80 @@ static const struct attribute_group *tdx_host_groups[] = { NULL, }; +static enum fw_upload_err tdx_fw_prepare(struct fw_upload *fwl, + const u8 *data, u32 data_len) +{ + return FW_UPLOAD_ERR_NONE; +} + +static enum fw_upload_err tdx_fw_write(struct fw_upload *fwl, const u8 *data, + u32 offset, u32 data_len, u32 *written) +{ + int ret; + + ret = seamldr_install_module(data, data_len); + switch (ret) { + case 0: + *written = data_len; + return FW_UPLOAD_ERR_NONE; + default: + return FW_UPLOAD_ERR_FW_INVALID; + } +} + +static enum fw_upload_err tdx_fw_poll_complete(struct fw_upload *fwl) +{ + /* + * The upload completed during tdx_fw_write(). + * Never poll for completion. + */ + return FW_UPLOAD_ERR_NONE; +} + +static void tdx_fw_cancel(struct fw_upload *fwl) +{ + /* + * TDX module updates are not cancellable. + * Provide a no-op callback to satisfy fw_upload_ops. + */ +} + +static const struct fw_upload_ops tdx_fw_ops = { + .prepare = tdx_fw_prepare, + .write = tdx_fw_write, + .poll_complete = tdx_fw_poll_complete, + .cancel = tdx_fw_cancel, +}; + +static void seamldr_deinit(void *tdx_fwl) +{ + firmware_upload_unregister(tdx_fwl); +} + +static int seamldr_init(struct device *dev) +{ + struct fw_upload *tdx_fwl; + + if (!supports_runtime_update()) + return 0; + + tdx_fwl = firmware_upload_register(THIS_MODULE, dev, "tdx_module", + &tdx_fw_ops, NULL); + if (IS_ERR(tdx_fwl)) + return PTR_ERR(tdx_fwl); + + return devm_add_action_or_reset(dev, seamldr_deinit, tdx_fwl); +} + +static int tdx_host_probe(struct faux_device *fdev) +{ + return seamldr_init(&fdev->dev); +} + +static const struct faux_device_ops tdx_host_ops = { + .probe = tdx_host_probe, +}; + static struct faux_device *fdev; static int __init tdx_host_init(void) @@ -127,7 +210,9 @@ static int __init tdx_host_init(void) if (!x86_match_cpu(tdx_host_ids) || !tdx_get_sysinfo()) return -ENODEV; - fdev = faux_device_create_with_groups(KBUILD_MODNAME, NULL, NULL, tdx_host_groups); + fdev = faux_device_create_with_groups(KBUILD_MODNAME, NULL, + &tdx_host_ops, + tdx_host_groups); if (!fdev) return -ENODEV; -- Gitee From 4dd00e2c9bd97ad1b4658eaf3d13c035dfea4025 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:01 -0700 Subject: [PATCH 54/78] x86/virt/seamldr: Allocate and populate a module update request ANBZ: #40036 commit 23a81e6cce154950ad203fd8bc5a016038c173b3 upstream. There are two important ABIs here: 'struct tdx_image' - The on-disk and in-memory format for a TDX module update image. 'struct seamldr_params' - The in-memory ABI passed to the TDX module loader. Points to a single 'struct tdx_image' broken up into 4k pages. Userspace supplies the update image in 'struct tdx_image' format. The image consists of a header followed by a sigstruct and the module binary. P-SEAMLDR, however, consumes 'struct seamldr_params' rather than the image directly. Parse the 'struct tdx_image' provided by userspace and populate a matching 'struct seamldr_params'. The 'tdx_image' ABI is versioned. Two public versions exist today: 0x100 and 0x200. This kernel only accepts 0x200. The older 0x100 format is being deprecated and is intentionally not supported here. Future versions of the module might be able to use the same ABIs (user/kernel and kernel/SEAMLDR) but they will not be able to use this kernel code. Reject module images without that specific version. This ensures that the kernel is able to understand the passed-in format. Validate the 'struct tdx_image' header before using it, because the header is consumed solely by the kernel to locate the sigstruct and module within the image. Do not validate the payload itself. The sigstruct and module pages are passed through to P-SEAMLDR, which validates them as part of the update. sigstruct_pages_pa_list currently has only one entry, but it will grow to four pages in the future. Keep it as an array for symmetry with module_pages_pa_list and for extensibility. [ dhansen: normal changelog clarification/munging ] backport: params = kzalloc_obj(*params); --> params = kzalloc(sizeof(*params), GFP_KERNEL); Intel-SIG: commit 23a81e6cce15 x86/virt/seamldr: Allocate and populate a module update request Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Link: https://patch.msgid.link/20260520133909.409394-14-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/seamldr.c | 157 +++++++++++++++++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index b2b6a439d417..9b8f8dace7d9 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -7,6 +7,8 @@ #define pr_fmt(fmt) "seamldr: " fmt #include +#include +#include #include #include @@ -18,6 +20,35 @@ /* P-SEAMLDR SEAMCALL leaf function */ #define P_SEAMLDR_INFO 0x8000000000000000 +#define SEAMLDR_MAX_NR_MODULE_PAGES 496 +#define SEAMLDR_MAX_NR_SIG_PAGES 1 + +/* + * The seamldr_params "scenario" field specifies the operation mode: + * 0: Install TDX module from scratch (not used by kernel) + * 1: Update existing TDX module to a compatible version + */ +#define SEAMLDR_SCENARIO_UPDATE 1 + +/* + * This is the "SEAMLDR_PARAMS" data structure defined in the + * "SEAM Loader (SEAMLDR) Interface Specification". + * + * It is the in-memory ABI that the kernel passes to the P-SEAMLDR + * to update the TDX module. It breaks the TDX module image up in + * page-size pieces. + */ +struct seamldr_params { + u32 version; + u32 scenario; + u64 sigstruct_pages_pa_list[SEAMLDR_MAX_NR_SIG_PAGES]; + u8 reserved[104]; + u64 module_nr_pages; + u64 module_pages_pa_list[SEAMLDR_MAX_NR_MODULE_PAGES]; +} __packed; + +static_assert(sizeof(struct seamldr_params) == 4096); + /* * Serialize P-SEAMLDR calls since the hardware only allows a single CPU to * interact with P-SEAMLDR simultaneously. Use raw version as the calls can @@ -55,6 +86,106 @@ int seamldr_get_info(struct seamldr_info *seamldr_info) } EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); +#define TDX_IMAGE_VERSION_2 0x200 + +/* First page of the on-disk module update image: */ +struct tdx_image_header { + u16 version; + u16 checksum; + u8 signature[8]; + u32 sigstruct_nr_pages; + u32 module_nr_pages; + u8 reserved[4076]; +} __packed; + +#define TDX_IMAGE_HEADER_SIZE sizeof(struct tdx_image_header) +static_assert(TDX_IMAGE_HEADER_SIZE == 4096); + +/* + * Intel TDX module update ABI structure. aka. "TDX module blob". + * This is the on-disk format that fw_upload lands in a kernel + * buffer. + * + * @payload contains sigstruct pages followed by module pages. + */ +struct tdx_image { + struct tdx_image_header header; + u8 payload[]; +}; + +/* + * Given a vmalloc() allocation, write all of the backing physical + * addresses to pa_list[]. Caller guarantees that the array is big + * enough. + */ +static void populate_pa_list(u64 *pa_list, const u8 *vmalloc_addr, u32 vmalloc_len_pages) +{ + int i; + + for (i = 0; i < vmalloc_len_pages; i++) { + unsigned long offset = i * PAGE_SIZE; + unsigned long pfn = vmalloc_to_pfn(&vmalloc_addr[offset]); + + pa_list[i] = pfn << PAGE_SHIFT; + } +} + +static void populate_seamldr_params(struct seamldr_params *params, + const u8 *sig, u32 sig_nr_pages, + const u8 *mod, u32 mod_nr_pages) +{ + params->version = 0; + params->scenario = SEAMLDR_SCENARIO_UPDATE; + params->module_nr_pages = mod_nr_pages; + + populate_pa_list(params->sigstruct_pages_pa_list, sig, sig_nr_pages); + populate_pa_list(params->module_pages_pa_list, mod, mod_nr_pages); +} + +/* + * @image points to a vmalloc()'d 'struct tdx_image'. Transform + * it into @params which is the P-SEAMLDR ABI format. + */ +static int init_seamldr_params(struct seamldr_params *params, + const struct tdx_image *image, + u32 image_len) +{ + const struct tdx_image_header *header = &image->header; + + u32 sigstruct_len = header->sigstruct_nr_pages * PAGE_SIZE; + u32 module_len = header->module_nr_pages * PAGE_SIZE; + + u8 *header_start = (u8 *)header; + u8 *header_end = header_start + TDX_IMAGE_HEADER_SIZE; + + u8 *sigstruct_start = header_end; + u8 *sigstruct_end = sigstruct_start + sigstruct_len; + + u8 *module_start = sigstruct_end; + + /* Check the calculated payload size against the image size. */ + if (TDX_IMAGE_HEADER_SIZE + sigstruct_len + module_len != image_len) + return -EINVAL; + + /* Reject unsupported tdx_image ABI versions. */ + if (header->version != TDX_IMAGE_VERSION_2) + return -EINVAL; + + if (header->sigstruct_nr_pages > SEAMLDR_MAX_NR_SIG_PAGES || + header->module_nr_pages > SEAMLDR_MAX_NR_MODULE_PAGES) + return -EINVAL; + + if (memcmp(header->signature, "TDX-BLOB", sizeof(header->signature))) + return -EINVAL; + + if (memchr_inv(header->reserved, 0, sizeof(header->reserved))) + return -EINVAL; + + populate_seamldr_params(params, sigstruct_start, header->sigstruct_nr_pages, + module_start, header->module_nr_pages); + return 0; +} + /** * seamldr_install_module - Install a new TDX module. * @data: Pointer to the TDX module image. @@ -64,7 +195,31 @@ EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); */ int seamldr_install_module(const u8 *data, u32 data_len) { + struct seamldr_params *params; + const struct tdx_image *image; + int ret; + + /* + * init_seamldr_params() reads the header early. + * Ensure there is enough data to do at least that. + */ + if (data_len < TDX_IMAGE_HEADER_SIZE) + return -EINVAL; + + image = (const struct tdx_image *)data; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + /* Populate 'params' from 'image'. */ + ret = init_seamldr_params(params, image, data_len); + if (ret) + goto out; + /* TODO: Update TDX module here */ - return 0; +out: + kfree(params); + return ret; } EXPORT_SYMBOL_FOR_MODULES(seamldr_install_module, "tdx-host"); -- Gitee From 2bd0eeb05bfac8c329e5e505bdd1629dd161f898 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:02 -0700 Subject: [PATCH 55/78] x86/virt/seamldr: Introduce skeleton for TDX module updates ANBZ: #40036 commit ab6be1168cf963630335a6f08938fd510a9225bf upstream. tl;dr: Use stop_machine() and a state machine based on the "MULTI_STOP" pattern to implement core TDX module update logic. Long version: TDX module updates require careful synchronization with other TDX operations. The requirements are (#1/#2 reflect current behavior that must be preserved): 1. SEAMCALLs need to be callable from both process and IRQ contexts. 2. SEAMCALLs need to be able to run concurrently across CPUs 3. During updates, only update-related SEAMCALLs are permitted; all other SEAMCALLs shouldn't be called. 4. During updates, all online CPUs must participate in the update work. No single lock primitive satisfies all requirements. For instance, rwlock_t handles #1/#2 but fails #4: CPUs spinning with IRQs disabled cannot be directed to perform update work. Use stop_machine() as it is the only well-understood mechanism that can meet all requirements. And TDX module updates consist of several steps (See Intel Trust Domain Extensions (Intel TDX) Module Base Architecture Specification, Chapter "TD-Preserving TDX module Update"). Ordering requirements between steps mandate lockstep synchronization across all CPUs. multi_cpu_stop() provides a good example of executing a multi-step task in lockstep across CPUs, but it does not synchronize the individual steps inside the callback itself. Implement a similar state machine as the skeleton for TDX module updates. Each state represents one step in the update flow, and the state advances only after all CPUs acknowledge completion of the current step. This acknowledgment mechanism provides the required lockstep execution. The update flow is intentionally simpler than multi_cpu_stop() in two ways: a) use a spinlock to protect the control data instead of atomic_t and explicit memory barriers. b) omit touch_nmi_watchdog() and rcu_momentary_eqs(), which exist there for debugging and are not strictly needed for this update flow Potential alternative to stop_machine() ======================================= An alternative approach is to lock all KVM entry points and kick all vCPUs. Here, KVM entry points refer to KVM VM/vCPU ioctl entry points, implemented in KVM common code (virt/kvm). Adding a locking mechanism there would affect all architectures KVM supports. And to lock only TDX vCPUs, new logic would be needed to identify TDX vCPUs, which the KVM common code currently lacks. This would add significant complexity and maintenance overhead to KVM for this TDX-specific use case, so don't take this approach. Intel-SIG: commit ab6be1168cf9 x86/virt/seamldr: Introduce skeleton for TDX module updates [ dhansen: normal changelog/style munging ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Xu Yilun Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-15-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/seamldr.c | 89 ++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 9b8f8dace7d9..67e753117fd2 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -8,8 +8,10 @@ #include #include +#include #include #include +#include #include #include @@ -186,6 +188,85 @@ static int init_seamldr_params(struct seamldr_params *params, return 0; } +/* + * During a TDX module update, all CPUs start from MODULE_UPDATE_START and + * progress to MODULE_UPDATE_DONE. Each state is associated with certain + * work. For some states, just one CPU needs to perform the work, while + * other CPUs just wait during those states. + */ +enum module_update_state { + MODULE_UPDATE_START, + MODULE_UPDATE_DONE, +}; + +static struct update_ctrl { + enum module_update_state state; + int num_ack; + /* + * Protect update_ctrl. Raw spinlock as it will be acquired from + * interrupt-disabled contexts. + */ + raw_spinlock_t lock; +} update_ctrl; + +/* Called with ctrl->lock held or during initialization. */ +static void __set_target_state(struct update_ctrl *ctrl, + enum module_update_state newstate) +{ + /* Reset ack counter. */ + ctrl->num_ack = 0; + ctrl->state = newstate; +} + +/* Last one to ack a state moves to the next state. */ +static void ack_state(struct update_ctrl *ctrl) +{ + raw_spin_lock(&ctrl->lock); + + ctrl->num_ack++; + if (ctrl->num_ack == num_online_cpus()) + __set_target_state(ctrl, ctrl->state + 1); + + raw_spin_unlock(&ctrl->lock); +} + +static void init_state(struct update_ctrl *ctrl) +{ + raw_spin_lock_init(&ctrl->lock); + __set_target_state(ctrl, MODULE_UPDATE_START + 1); +} + +/* + * See multi_cpu_stop() from where this multi-cpu state-machine was + * adopted. + */ +static int do_seamldr_install_module(void *seamldr_params) +{ + enum module_update_state curstate = MODULE_UPDATE_START; + enum module_update_state newstate; + int ret = 0; + + do { + newstate = READ_ONCE(update_ctrl.state); + + if (curstate == newstate) { + cpu_relax(); + continue; + } + + curstate = newstate; + switch (curstate) { + /* TODO: add the update steps. */ + default: + break; + } + + ack_state(&update_ctrl); + } while (curstate != MODULE_UPDATE_DONE); + + return ret; +} + /** * seamldr_install_module - Install a new TDX module. * @data: Pointer to the TDX module image. @@ -217,7 +298,13 @@ int seamldr_install_module(const u8 *data, u32 data_len) if (ret) goto out; - /* TODO: Update TDX module here */ + /* Ensure a stable set of online CPUs for the update process. */ + cpus_read_lock(); + init_state(&update_ctrl); + ret = stop_machine_cpuslocked(do_seamldr_install_module, params, + cpu_online_mask); + cpus_read_unlock(); + out: kfree(params); return ret; -- Gitee From 974d3f0eada1bc8446a4a2bb865e94135ae26943 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:04 -0700 Subject: [PATCH 56/78] x86/virt/seamldr: Abort updates after a failed step ANBZ: #40036 commit be4efe63c050be48961a5430c91e69f95af08b81 upstream. A TDX module update is a multi-step process, and any step can fail. The current update flow continues to later steps after an error. Continuing after a failure can cause the TDX module to enter an unrecoverable state. But certain failures during the initial module shutdown step should simply return an error to userspace, so the update can be retried cleanly. To preserve that recoverability, one option would be to abort the update only for those failures, since they occur before any TDX module state is changed. But special-casing specific failures in specific steps would complicate the do-while() update loop for no benefit. Simply abort update on any failure, at any step. Track failures for each step, stop the update loop once a failure is observed, and do not advance the state machine to the next step. Intel-SIG: commit be4efe63c050 x86/virt/seamldr: Abort updates after a failed step [ dhansen: style nits ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Xu Yilun Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Link: https://lore.kernel.org/linux-coco/aQFmOZCdw64z14cJ@google.com/ # [1] Link: https://patch.msgid.link/20260520133909.409394-16-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/seamldr.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 67e753117fd2..448d7a83f5b7 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -202,6 +202,7 @@ enum module_update_state { static struct update_ctrl { enum module_update_state state; int num_ack; + int num_failed; /* * Protect update_ctrl. Raw spinlock as it will be acquired from * interrupt-disabled contexts. @@ -219,12 +220,13 @@ static void __set_target_state(struct update_ctrl *ctrl, } /* Last one to ack a state moves to the next state. */ -static void ack_state(struct update_ctrl *ctrl) +static void ack_state(struct update_ctrl *ctrl, int result) { raw_spin_lock(&ctrl->lock); + ctrl->num_failed += !!result; ctrl->num_ack++; - if (ctrl->num_ack == num_online_cpus()) + if (ctrl->num_ack == num_online_cpus() && !ctrl->num_failed) __set_target_state(ctrl, ctrl->state + 1); raw_spin_unlock(&ctrl->lock); @@ -234,6 +236,7 @@ static void init_state(struct update_ctrl *ctrl) { raw_spin_lock_init(&ctrl->lock); __set_target_state(ctrl, MODULE_UPDATE_START + 1); + ctrl->num_failed = 0; } /* @@ -261,8 +264,9 @@ static int do_seamldr_install_module(void *seamldr_params) break; } - ack_state(&update_ctrl); - } while (curstate != MODULE_UPDATE_DONE); + ack_state(&update_ctrl, ret); + } while (curstate != MODULE_UPDATE_DONE && + !READ_ONCE(update_ctrl.num_failed)); return ret; } -- Gitee From 244fb3e4c5c76b4d2b30a347ff57c44fd22ba22b Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:05 -0700 Subject: [PATCH 57/78] x86/virt/seamldr: Shut down the current TDX module ANBZ: #40036 commit 9ea06080a680a2aa521795432396d2c27f9bb9f9 upstream. The first step of TDX module updates is shutting down the current TDX module. This step also packs state information that needs to be preserved across updates, called "handoff data". This handoff data is consumed by the updated module and stored internally in the SEAM range and hidden from the kernel. Since the handoff data layout may change between modules, the handoff data is versioned. Each module has a native handoff version and provides backward support for several older versions. The complete handoff versioning protocol is complex as it supports both module upgrades and downgrades. See details in "Intel Trust Domain Extensions (Intel TDX) Module Base Architecture Specification", Chapter "Handoff Versioning". Ideally, the kernel needs to retrieve the handoff versions supported by the current module and the new module and select a version supported by both. But since this implementation only supports module upgrades, simply request handoff data from the current module using its highest supported version. That is sufficient for this upgrade-only implementation. Retrieve the module's handoff version from TDX global metadata and add an update step to shut down the module. Module shutdown only needs to run on one CPU. Don't cache the handoff information in tdx_sysinfo. It is used only for module shutdown, and is present only when the TDX module supports updates. Caching it in get_tdx_sys_info() would require extra update-support guards and refreshing the cached value across module updates. [ dhansen: fix up function variables, remove 'cpu'. Return from tdx_module_shutdown() early if handoff call fails. ] Intel-SIG: commit 9ea06080a680 x86/virt/seamldr: Shut down the current TDX module Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Tony Lindgren Reviewed-by: Xu Yilun Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Link: https://patch.msgid.link/20260520133909.409394-17-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx_global_metadata.h | 4 ++++ arch/x86/virt/vmx/tdx/seamldr.c | 15 ++++++++++++- arch/x86/virt/vmx/tdx/tdx.c | 25 ++++++++++++++++++++- arch/x86/virt/vmx/tdx/tdx.h | 3 +++ arch/x86/virt/vmx/tdx/tdx_global_metadata.c | 13 +++++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/tdx_global_metadata.h b/arch/x86/include/asm/tdx_global_metadata.h index 40689c8dc67e..41150d546589 100644 --- a/arch/x86/include/asm/tdx_global_metadata.h +++ b/arch/x86/include/asm/tdx_global_metadata.h @@ -40,6 +40,10 @@ struct tdx_sys_info_td_conf { u64 cpuid_config_values[128][2]; }; +struct tdx_sys_info_handoff { + u16 module_hv; +}; + struct tdx_sys_info { struct tdx_sys_info_version version; struct tdx_sys_info_features features; diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 448d7a83f5b7..fe124d3351f9 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -18,6 +18,7 @@ #include #include "seamcall_internal.h" +#include "tdx.h" /* P-SEAMLDR SEAMCALL leaf function */ #define P_SEAMLDR_INFO 0x8000000000000000 @@ -196,6 +197,7 @@ static int init_seamldr_params(struct seamldr_params *params, */ enum module_update_state { MODULE_UPDATE_START, + MODULE_UPDATE_SHUTDOWN, MODULE_UPDATE_DONE, }; @@ -247,8 +249,16 @@ static int do_seamldr_install_module(void *seamldr_params) { enum module_update_state curstate = MODULE_UPDATE_START; enum module_update_state newstate; + bool is_lead_cpu = false; int ret = 0; + /* + * Some steps must be run on exactly one CPU. Pick a "lead" CPU to + * execute those steps. Use CPU 0 because it is always online. + */ + if (smp_processor_id() == 0) + is_lead_cpu = true; + do { newstate = READ_ONCE(update_ctrl.state); @@ -259,7 +269,10 @@ static int do_seamldr_install_module(void *seamldr_params) curstate = newstate; switch (curstate) { - /* TODO: add the update steps. */ + case MODULE_UPDATE_SHUTDOWN: + if (is_lead_cpu) + ret = tdx_module_shutdown(); + break; default: break; } diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 8b07b7c1011e..eb30f99f6a1f 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -295,7 +295,7 @@ static __init int build_tdx_memlist(struct list_head *tmb_list) return ret; } -static __init int read_sys_metadata_field(u64 field_id, u64 *data) +static int read_sys_metadata_field(u64 field_id, u64 *data) { struct tdx_module_args args = {}; int ret; @@ -1241,6 +1241,29 @@ static __init int tdx_enable(void) } subsys_initcall(tdx_enable); +int tdx_module_shutdown(void) +{ + struct tdx_sys_info_handoff handoff = {}; + struct tdx_module_args args = {}; + int ret; + + ret = get_tdx_sys_info_handoff(&handoff); + /* + * Handoff information is required for proper + * shutdown. Refuse to shut down without it. + */ + if (ret) + return ret; + + /* + * Use the module's handoff version as it is the highest the + * module can produce and most likely supported by newer modules. + */ + args.rcx = handoff.module_hv; + + return seamcall_prerr(TDH_SYS_SHUTDOWN, &args); +} + static bool is_pamt_page(unsigned long phys) { struct tdmr_info_list *tdmr_list = &tdx_tdmr_list; diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index 76c5fb1e1ffe..f0c20dea0388 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -46,6 +46,7 @@ #define TDH_PHYMEM_PAGE_WBINVD 41 #define TDH_VP_WR 43 #define TDH_SYS_CONFIG 45 +#define TDH_SYS_SHUTDOWN 52 #define TDH_SYS_DISABLE 69 /* @@ -108,4 +109,6 @@ struct tdmr_info_list { int max_tdmrs; /* How many 'tdmr_info's are allocated */ }; +int tdx_module_shutdown(void); + #endif diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c index 982b7471e517..e49c300f23d4 100644 --- a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -100,6 +100,19 @@ static __init int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_ return ret; } +static int get_tdx_sys_info_handoff(struct tdx_sys_info_handoff *sysinfo_handoff) +{ + int ret; + u64 val; + + ret = read_sys_metadata_field(0x8900000100000000, &val); + if (ret) + return ret; + + sysinfo_handoff->module_hv = val; + return 0; +} + static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo) { int ret = 0; -- Gitee From 0684ba8a54ac278da8fc739c9e6c2934a6b6397e Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:06 -0700 Subject: [PATCH 58/78] x86/virt/tdx: Reset software states during TDX module shutdown ANBZ: #40036 commit 65a6542a8144cb88ffccae386e38d61933d575c8 upstream. The TDX module requires a one-time global initialization (TDH.SYS.INIT) and per-CPU initialization (TDH.SYS.LP.INIT) before use. These initializations are guarded by software flags to prevent repetition. Reset all software flags guarding the initialization flows to allow the global and per-CPU initializations to be triggered again after updates. Intel-SIG: commit 65a6542a8144 x86/virt/tdx: Reset software states during TDX module shutdown [ dhansen: trim down changelog ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-18-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index eb30f99f6a1f..231c64bbde98 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1246,6 +1246,7 @@ int tdx_module_shutdown(void) struct tdx_sys_info_handoff handoff = {}; struct tdx_module_args args = {}; int ret; + int cpu; ret = get_tdx_sys_info_handoff(&handoff); /* @@ -1261,7 +1262,21 @@ int tdx_module_shutdown(void) */ args.rcx = handoff.module_hv; - return seamcall_prerr(TDH_SYS_SHUTDOWN, &args); + ret = seamcall_prerr(TDH_SYS_SHUTDOWN, &args); + if (ret) + return ret; + + /* + * Clear global and per-CPU initialization flags so the new module + * can be fully re-initialized after a successful update. + * + * No locks needed as no concurrent accesses can occur here. + */ + memset(&tdx_module_state, 0, sizeof(tdx_module_state)); + for_each_possible_cpu(cpu) + per_cpu(tdx_lp_initialized, cpu) = false; + + return 0; } static bool is_pamt_page(unsigned long phys) -- Gitee From 11f1aa1b0efabf8ecb86407882146f03bbd5401b Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:08 -0700 Subject: [PATCH 59/78] x86/virt/seamldr: Install a new TDX module ANBZ: #40036 commit 2bfb2ef877f510bc6ebe8a74ce0877d290dc8bcd upstream. Continue fleshing out the update proces. The old module is shut down and the system is ready for the new module image. Run the SEAMLDR.INSTALL SEAMCALL on all CPUs. Intel-SIG: commit 2bfb2ef877f5 x86/virt/seamldr: Install a new TDX module Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Xu Yilun Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-19-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/seamldr.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index fe124d3351f9..a65c3c3053c5 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -22,6 +22,7 @@ /* P-SEAMLDR SEAMCALL leaf function */ #define P_SEAMLDR_INFO 0x8000000000000000 +#define P_SEAMLDR_INSTALL 0x8000000000000001 #define SEAMLDR_MAX_NR_MODULE_PAGES 496 #define SEAMLDR_MAX_NR_SIG_PAGES 1 @@ -89,6 +90,15 @@ int seamldr_get_info(struct seamldr_info *seamldr_info) } EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); +/* Call into P-SEAMLDR to install a TDX module update */ +static int seamldr_install(const struct seamldr_params *params) +{ + struct tdx_module_args args = {}; + + args.rcx = __pa(params); + return seamldr_call(P_SEAMLDR_INSTALL, &args); +} + #define TDX_IMAGE_VERSION_2 0x200 /* First page of the on-disk module update image: */ @@ -198,6 +208,7 @@ static int init_seamldr_params(struct seamldr_params *params, enum module_update_state { MODULE_UPDATE_START, MODULE_UPDATE_SHUTDOWN, + MODULE_UPDATE_CPU_INSTALL, MODULE_UPDATE_DONE, }; @@ -273,6 +284,9 @@ static int do_seamldr_install_module(void *seamldr_params) if (is_lead_cpu) ret = tdx_module_shutdown(); break; + case MODULE_UPDATE_CPU_INSTALL: + ret = seamldr_install(seamldr_params); + break; default: break; } -- Gitee From 244be57f5490fcc559e4b1098249d5e24432e2fe Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:09 -0700 Subject: [PATCH 60/78] x86/virt/seamldr: Initialize the newly-installed TDX module ANBZ: #40036 commit bf7c0ed2c3621f4f0bf56efcd95e5a5372bf0ca6 upstream. Continue fleshing out the update process. At this point the new module is sitting in memory but has never been called and is not usable. It is in a similar state to the when the system first boots. Leave the P-SEAMLDR behind. Stop making calls to it. Transition to calling the new TDX module itself to set up both global and per-cpu state. Share tdx_cpu_enable() with the fresh-boot module initialization code. Export it and invoke it on all CPUs. Note: "TDX global initialization" needs to be done once before "TDX per-CPU initialization". It would be a great fit for the new runtime update "is_lead_cpu" logic. But tdx_cpu_enable() already has some logic to do the global initialization properly. Just use it directly to maximize fresh-boot and runtime update code sharing. == Background == The boot-time and post-update initialization flows share the same first steps: - TDX global initialization - TDX per-CPU initialization After that, they diverge: - Fresh boot: Prepare TDMRs/PAMTs Configure the TDX module Configure the global KeyID Initialize TDMRs - Runtime update: Restore TDX module state from handoff data Future changes will consume the handoff data. Intel-SIG: commit bf7c0ed2c362 x86/virt/seamldr: Initialize the newly-installed TDX module [ dhansen: major changelog munging ] Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Xu Yilun Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-20-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx.h | 2 +- arch/x86/virt/vmx/tdx/seamldr.c | 4 ++++ arch/x86/virt/vmx/tdx/tdx.c | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index f27fbd9b11aa..852441da70fe 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -107,7 +107,7 @@ static inline long tdx_kvm_hypercall(unsigned int nr, unsigned long p1, #ifdef CONFIG_INTEL_TDX_HOST void tdx_init(void); - +int tdx_cpu_enable(void); const char *tdx_dump_mce_info(struct mce *m); const struct tdx_sys_info *tdx_get_sysinfo(void); diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index a65c3c3053c5..aa39f6b995c9 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -209,6 +209,7 @@ enum module_update_state { MODULE_UPDATE_START, MODULE_UPDATE_SHUTDOWN, MODULE_UPDATE_CPU_INSTALL, + MODULE_UPDATE_CPU_INIT, MODULE_UPDATE_DONE, }; @@ -287,6 +288,9 @@ static int do_seamldr_install_module(void *seamldr_params) case MODULE_UPDATE_CPU_INSTALL: ret = seamldr_install(seamldr_params); break; + case MODULE_UPDATE_CPU_INIT: + ret = tdx_cpu_enable(); + break; default: break; } diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 231c64bbde98..b25f2341d56d 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -112,7 +112,7 @@ static int try_init_module_global(void) * (and TDX module global initialization SEAMCALL if not done) on local cpu to * make this cpu be ready to run any other SEAMCALLs. */ -static int tdx_cpu_enable(void) +int tdx_cpu_enable(void) { struct tdx_module_args args = {}; int ret; -- Gitee From 7a2a44d1b5cd889419927076b4d74c28dcde9b46 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:10 -0700 Subject: [PATCH 61/78] x86/virt/tdx: Restore TDX module state ANBZ: #40036 commit b344c50a1ad4230c86bcc080a08ee65e8c944627 upstream. After per-CPU initialization, the module is nearly functional. It is in a similar state to TDX initialization before TDH.SYS.CONFIG. At this point, the kernel _could_ just repeat the boot-time sequence, but that would land the new module in a slightly different state than the old module. This would leave old TDs unrunnable, which is not a good outcome. Thankfully, the "handoff" data saved during module shutdown should contain all the information needed to restore the TDX module state to exactly what it was before the update. Restore TDX module state. The TDX module only needs a single copy so only do this on the lead CPU. Restoration errors can theoretically be handled in a few ways. For instance, userspace could try to load a different TDX module version. Or, the kernel could give up on the handoff process and just reinitialize the new module from scratch, which would lose all existing TDs. Simply propagate errors to userspace. Ignore the idea of a TD-destroying reinitialization. It would destroy data like a reboot and if things have gone that wrong a reboot is probably the best option anyway. Note: the location and the format of handoff data is defined by the TDX module. The new module knows where to get handoff data and how to parse it. The kernel does not touch it at all. Intel-SIG: commit b344c50a1ad4 x86/virt/tdx: Restore TDX module state Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Tony Lindgren Reviewed-by: Kai Huang Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-21-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/seamldr.c | 5 +++++ arch/x86/virt/vmx/tdx/tdx.c | 13 +++++++++++++ arch/x86/virt/vmx/tdx/tdx.h | 2 ++ 3 files changed, 20 insertions(+) diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index aa39f6b995c9..9a00f0b870ec 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -210,6 +210,7 @@ enum module_update_state { MODULE_UPDATE_SHUTDOWN, MODULE_UPDATE_CPU_INSTALL, MODULE_UPDATE_CPU_INIT, + MODULE_UPDATE_RUN_UPDATE, MODULE_UPDATE_DONE, }; @@ -291,6 +292,10 @@ static int do_seamldr_install_module(void *seamldr_params) case MODULE_UPDATE_CPU_INIT: ret = tdx_cpu_enable(); break; + case MODULE_UPDATE_RUN_UPDATE: + if (is_lead_cpu) + ret = tdx_module_run_update(); + break; default: break; } diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index b25f2341d56d..02e96d165ba2 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -1279,6 +1279,19 @@ int tdx_module_shutdown(void) return 0; } +int tdx_module_run_update(void) +{ + struct tdx_module_args args = {}; + int ret; + + ret = seamcall_prerr(TDH_SYS_UPDATE, &args); + if (ret) + return ret; + + tdx_module_state.initialized = true; + return 0; +} + static bool is_pamt_page(unsigned long phys) { struct tdmr_info_list *tdmr_list = &tdx_tdmr_list; diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h index f0c20dea0388..bdfd0e1e337a 100644 --- a/arch/x86/virt/vmx/tdx/tdx.h +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -47,6 +47,7 @@ #define TDH_VP_WR 43 #define TDH_SYS_CONFIG 45 #define TDH_SYS_SHUTDOWN 52 +#define TDH_SYS_UPDATE 53 #define TDH_SYS_DISABLE 69 /* @@ -110,5 +111,6 @@ struct tdmr_info_list { }; int tdx_module_shutdown(void); +int tdx_module_run_update(void); #endif -- Gitee From 4dfb7391d04e53aa7ea67141fe66b17771620ac4 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Fri, 22 May 2026 08:43:02 -0700 Subject: [PATCH 62/78] x86/virt/seamldr: Add module update locking ANBZ: #40036 commit 6e97c234cdf0d11f51f75e918b3f612e57ecd027 upstream. TDX metadata like the version number changes during a module update. Add functions to lock out module updates. The current stop_machine() implementation uses worker threads. The scheduler actually does a full, normal context switch over to that thread. preempt_disable() obviously inhibits that context switch and thus, locks out stop_machine() users like the module update. Thanks to Chao for the idea of using preempt_disable(). Intel-SIG: commit 6e97c234cdf0 x86/virt/seamldr: Add module update locking Signed-off-by: Dave Hansen [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/seamldr.h | 2 ++ arch/x86/virt/vmx/tdx/seamldr.c | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/arch/x86/include/asm/seamldr.h b/arch/x86/include/asm/seamldr.h index 43084e2daa2d..cfc6a1b1a440 100644 --- a/arch/x86/include/asm/seamldr.h +++ b/arch/x86/include/asm/seamldr.h @@ -32,5 +32,7 @@ static_assert(sizeof(struct seamldr_info) == 256); int seamldr_get_info(struct seamldr_info *seamldr_info); int seamldr_install_module(const u8 *data, u32 data_len); +void seamldr_lock_module_update(void); +void seamldr_unlock_module_update(void); #endif /* _ASM_X86_SEAMLDR_H */ diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c index 9a00f0b870ec..d675038999c9 100644 --- a/arch/x86/virt/vmx/tdx/seamldr.c +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -350,3 +350,19 @@ int seamldr_install_module(const u8 *data, u32 data_len) return ret; } EXPORT_SYMBOL_FOR_MODULES(seamldr_install_module, "tdx-host"); + +/* + * stop_machine() does not interrupt preemption-disabled regions. + * Simply disabling preempt prevents updates. + */ +void seamldr_lock_module_update(void) +{ + preempt_disable(); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_lock_module_update, "tdx-host"); + +void seamldr_unlock_module_update(void) +{ + preempt_enable(); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_unlock_module_update, "tdx-host"); -- Gitee From 710e499e74261ad5691f6b57c52475232c0a22d5 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Fri, 22 May 2026 08:56:29 -0700 Subject: [PATCH 63/78] coco/tdx-host: Lock out module updates when reading version ANBZ: #40036 commit 8d1376e6b0e6d98bed8e7d14fa8587a1d1139e93 upstream. The TDX module version is currently stashed in some global variables and dumped out to sysfs without locking. This works fine when the version is static and never changes. But with runtime module updates, the TDX module version can change. Some kind of locking is needed. Barring this, userspace could theoretically see a strange torn module version that is some Frankenstein version from from two different updates. Use the new module update lock/unlock to prevent updates while trying to read the version. Don't be fussy about it. There's no need to snapshot the version or do READ_ONCE(), or minimize lock holding times. sysfs_emit() does not sleep. Also note that the lock/unlock are backed by preempt_dis/enable() which are really cheap CPU-local operations. This is not a heavyweight lock. Intel-SIG: commit 8d1376e6b0e6 coco/tdx-host: Lock out module updates when reading version Signed-off-by: Dave Hansen [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- drivers/virt/coco/tdx-host/tdx-host.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/virt/coco/tdx-host/tdx-host.c b/drivers/virt/coco/tdx-host/tdx-host.c index e5a672be6200..d48952968e86 100644 --- a/drivers/virt/coco/tdx-host/tdx-host.c +++ b/drivers/virt/coco/tdx-host/tdx-host.c @@ -26,15 +26,24 @@ static ssize_t version_show(struct device *dev, struct device_attribute *attr, { const struct tdx_sys_info *tdx_sysinfo = tdx_get_sysinfo(); const struct tdx_sys_info_version *ver; + int ret; if (!tdx_sysinfo) return -ENXIO; + /* + * The version number can change during an update. + * Lock out updates while printing the version. + */ + seamldr_lock_module_update(); + ver = &tdx_sysinfo->version; + ret = sysfs_emit(buf, TDX_VERSION_FMT "\n", ver->major_version, + ver->minor_version, + ver->update_version); + seamldr_unlock_module_update(); - return sysfs_emit(buf, TDX_VERSION_FMT "\n", ver->major_version, - ver->minor_version, - ver->update_version); + return ret; } static DEVICE_ATTR_RO(version); -- Gitee From db640b4a174fb95adbcb6d9d86d6ca191ebfd6c0 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:12 -0700 Subject: [PATCH 64/78] x86/virt/tdx: Refresh TDX module version after update ANBZ: #40036 commit 856bc8bb990e21546fadff83502cdad33660a18e upstream. The kernel exposes the TDX module version through sysfs so userspace can check update compatibility. That information needs to remain accurate across runtime updates. A runtime update may change the module's update_version, so refresh the cached version right after a successful update. Drop __ro_after_init from tdx_sysinfo because it is now updated at runtime. Do not refresh the rest of tdx_sysinfo, even if some values change across updates. TDX module updates are backward compatible, so existing tdx_sysinfo consumers, such as KVM, can continue to operate without seeing the new values. [ dhansen: trim changelog ] backport: merged have this change -static __init int get_tdx_sys_info_version(struct tdx_sys_info_version *sysinfo_version) +static int get_tdx_sys_info_version(struct tdx_sys_info_version *sysinfo_version) Intel-SIG: commit 856bc8bb990e x86/virt/tdx: Refresh TDX module version after update Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-22-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/vmx/tdx/tdx.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c index 02e96d165ba2..898be1de7fe1 100644 --- a/arch/x86/virt/vmx/tdx/tdx.c +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -66,7 +66,7 @@ static struct tdmr_info_list tdx_tdmr_list; /* All TDX-usable memory regions. Protected by mem_hotplug_lock. */ static LIST_HEAD(tdx_memlist); -static struct tdx_sys_info tdx_sysinfo __ro_after_init; +static struct tdx_sys_info tdx_sysinfo; static DEFINE_RAW_SPINLOCK(sysinit_lock); @@ -1288,6 +1288,13 @@ int tdx_module_run_update(void) if (ret) return ret; + ret = get_tdx_sys_info_version(&tdx_sysinfo.version); + /* + * Only fails if there is something unexpected + * and severely wrong with the module. + */ + WARN_ON_ONCE(ret); + tdx_module_state.initialized = true; return 0; } -- Gitee From 7f94f6119727ae1df8321d19571f9bbfa99bd241 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Wed, 20 May 2026 15:29:13 -0700 Subject: [PATCH 65/78] x86/virt/tdx: Enable TDX module runtime updates ANBZ: #40036 commit 1e35f5945db9630c0602d9a9a7bbd146a96d13cc upstream. All pieces of TDX module runtime updates are in place. Enable it if it is supported. Intel-SIG: commit 1e35f5945db9 x86/virt/tdx: Enable TDX module runtime updates Signed-off-by: Chao Gao Signed-off-by: Dave Hansen Reviewed-by: Xu Yilun Reviewed-by: Tony Lindgren Reviewed-by: Kiryl Shutsemau (Meta) Reviewed-by: Rick Edgecombe Link: https://patch.msgid.link/20260520133909.409394-24-chao.gao@intel.com [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/include/asm/tdx.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h index 852441da70fe..99f488d6e814 100644 --- a/arch/x86/include/asm/tdx.h +++ b/arch/x86/include/asm/tdx.h @@ -33,6 +33,7 @@ #define TDX_RND_NO_ENTROPY 0x8000020300000000ULL /* Bit definitions of TDX_FEATURES0 metadata field */ +#define TDX_FEATURES0_TD_PRESERVING BIT_ULL(1) #define TDX_FEATURES0_NO_RBP_MOD BIT_ULL(18) #ifndef __ASSEMBLY__ @@ -113,8 +114,7 @@ const struct tdx_sys_info *tdx_get_sysinfo(void); static inline bool tdx_supports_runtime_update(const struct tdx_sys_info *sysinfo) { - /* To be enabled when kernel is ready. */ - return false; + return sysinfo->features.tdx_features0 & TDX_FEATURES0_TD_PRESERVING; } int tdx_guest_keyid_alloc(void); -- Gitee From 89ec8d169569e651f7c6043ec51cf4f3a28413c2 Mon Sep 17 00:00:00 2001 From: Chao Gao Date: Fri, 22 May 2026 14:41:25 -0700 Subject: [PATCH 66/78] x86/virt/tdx: Document TDX module update ANBZ: #40036 commit 2b9ad7a6154e0938b9458691536296dd0224942d upstream. Recent changes introduced TDX module update support. This is a thorny feature to use correctly. It is intended for informed admins only who are expected to either be doing things very carefully, or using it via userspace programs that handle the pitfalls for them. Document the basics of how to use the feature and what is expected of the user in order for it to go correctly. Both to help the intended users of the feature and as a "here be dragons" note for the more casual TDX users. [ dhansen: tweak docs a bit, clarify update constraints ] Signed-off-by: Chao Gao [dropped "Implementation details" section, update log] Intel-SIG: commit 2b9ad7a6154e x86/virt/tdx: Document TDX module update Signed-off-by: Rick Edgecombe Signed-off-by: Dave Hansen [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Documentation/arch/x86/tdx.rst | 127 +++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst index 628542521b89..0fa381e46ea8 100644 --- a/Documentation/arch/x86/tdx.rst +++ b/Documentation/arch/x86/tdx.rst @@ -73,6 +73,133 @@ initialize:: [..] virt/tdx: TDX-Module initialization failed ... +TDX module Runtime Update +------------------------- + +Similar to microcode, the BIOS generally has a copy of the TDX module +in flash. It loads this module image in to RAM at boot. However, just +like microcode, the BIOS-loaded TDX module might be out of date either +because the BIOS is old or the system has been up a long time. The +kernel can replace the BIOS version in RAM and load a different TDX +module. Kernel-loaded TDX modules do not affect the BIOS flash and do +not survive reboots. + +The TDX module is normally the only piece of software running in SEAM +mode with which the kernel interacts. But there is a second piece of +software which is used to load or update the TDX module: a persistent +SEAM loader (P-SEAMLDR). It runs in SEAM mode separately from the TDX +module. The kernel communicates with the P-SEAMLDR to perform TDX +module runtime updates. + +How to update the TDX module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Updating the TDX module is a complex process. Much of the logic and +policy is left to userspace. End users should use existing update +infrastructure provided by their distro. The Intel TDX Module Binaries +repository has a reference implementation of this logic: + + https://github.com/intel/confidential-computing.tdx.tdx-module.binaries/blob/main/version_select_and_load.py + +This section will now lay out roughly what is needed to implement a +userspace-driven TDX module update. Detailed documentation on the +tdx_host ABIs is available here:: + + Documentation/ABI/testing/sysfs-devices-faux-tdx-host + +and is not duplicated in this document. + +1. Check whether runtime update is supported at all + + Verify that the TDX module firmware upload interface is available:: + + /sys/class/firmware/tdx_module + + Note that this is the generic kernel firmware update ABI. It is + separate from the "tdx_host" device ABI itself. + +2. Check whether additional updates are possible. Verify that:: + + /sys/devices/faux/tdx_host/num_remaining_updates + + has a value greater than 0. If it is 0, the TDX update log might be + full. Reboot to reset this to a nonzero value. + +3. Choose a compatible TDX module image + + Choosing a compatible TDX module image is not trivial. There are both + hard compatibility requirements and policy choices to make. + + Hard compatibility requirements: + + - The update must be compatible with the kernel. + + The update must not change any TDX ABIs in any non-backward-compatible + way. It can introduce new features but must not require that the kernel + use new ABIs for existing features. It must ensure that the rest of the + system is not affected in any way. Software on the system must never + notice any behavioral changes. Attestation results should be identical + except for version changes. + + - The update must be compatible with the CPU. + + The set of supported CPU FMS values (family, model, stepping) is + encoded in the module image itself. In practice, module version series + are platform-specific. For example, the 1.5.x series runs on Sapphire + Rapids but not Granite Rapids, which needs 2.0.x. + + - The update must be compatible with the P-SEAMLDR. + + This information is provided in a metadata file, typically + mapping_file.json, released with the module image. Each module image + specifies the minimum required P-SEAMLDR version, and the update is + compatible only if the running P-SEAMLDR meets that requirement. + + The current version of the P-SEAMLDR can be read here:: + + /sys/devices/faux/tdx_host/seamldr_version + + - The update must be compatible with the running TDX module. + + Like P-SEAMLDR, each module image also specifies a minimum required + TDX module version. The running module must satisfy that requirement. + + The update software can read the current TDX module version here:: + + /sys/devices/faux/tdx_host/version + + Policy choices: + + - The update software chooses how to optimize its update. For instance, + it can optimize for fewer updates or for smaller version steps, + for example, 1.2.3 => 1.2.5 versus 1.2.3 => 1.2.4 => 1.2.5. + +4. Perform the update + + Run:: + + echo 1 > /sys/class/firmware/tdx_module/loading + cat > /sys/class/firmware/tdx_module/data + echo 0 > /sys/class/firmware/tdx_module/loading + + The files /sys/class/firmware/tdx_module/status and + /sys/class/firmware/tdx_module/error report update progress and error + information. + + After the update completes, the new module version is visible in + /sys/devices/faux/tdx_host/version. + +Impact on running TDs +~~~~~~~~~~~~~~~~~~~~~ + +TDX module runtime updates must have virtually no visible impact on running +TDs. Any TD visible impact is a TDX module bug. + +The main exception is the TEE_TCB_SVN_2 field in TD quotes, which +reflects the TCB of the currently running TDX module and therefore +changes after an update. By contrast, TEE_TCB_SVN reflects the TCB at TD +launch time and is not affected. + TDX Interaction to Other Kernel Components ------------------------------------------ -- Gitee From 6265f9644bf7d460ea08e3531711f63fd4f85b9e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 22 May 2026 15:46:06 -0700 Subject: [PATCH 67/78] KVM: SEV: Pin source page for write when adding CPUID data for SNP guest ANBZ: #40036 commit f13e900599089b10113ceb36013423f0837c6792 upstream. When populating a guest_memfd instance with the initial CPUID data for an SNP guest, acquire a writable pin on the source page as KVM will write back the "correct" CPUID information if the userspace provided data is rejected by trusted firmware. Because KVM writes to the source page using a kernel mapping, pinning for read could result in KVM clobbering read-only memory. Note, well-behaved VMMs are unlikely to be affected, as CPUID information is almost always dynamically generated by userspace, i.e. it's unlikely for the CPUID information to be backed by a read-only mapping. Intel-SIG: commit f13e90059908 KVM: SEV: Pin source page for write when adding CPUID data for SNP guest Fixes: 2a62345b30529 ("KVM: guest_memfd: GUP source pages prior to populating guest memory") Cc: stable@vger.kernel.org Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260522-fix-sev-gmem-post-populate-v2-1-3f196bfad5a1@google.com [sean: rewrite shortlog and changelog, tag for stable@] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 1 + arch/x86/kvm/vmx/tdx.c | 2 +- include/linux/kvm_host.h | 3 ++- virt/kvm/guest_memfd.c | 6 ++++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 2078e683c11c..00278e3a36ee 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2512,6 +2512,7 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) sev_populate_args.type = params.type; count = kvm_gmem_populate(kvm, params.gfn_start, src, npages, + params.type == KVM_SEV_SNP_PAGE_TYPE_CPUID, sev_gmem_post_populate, &sev_populate_args); if (count < 0) { argp->error = sev_populate_args.fw_error; diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 097ef17559a1..9979355024e2 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3199,7 +3199,7 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c }; gmem_ret = kvm_gmem_populate(kvm, gpa_to_gfn(region.gpa), u64_to_user_ptr(region.source_addr), - 1, tdx_gmem_post_populate, &arg); + 1, false, tdx_gmem_post_populate, &arg); if (gmem_ret < 0) { ret = gmem_ret; break; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index fdb90587e22d..31fbd70332ce 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2641,7 +2641,8 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, struct page *page, void *opaque); -long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, +long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, + long npages, bool may_writeback_src, kvm_gmem_populate_cb post_populate, void *opaque); #endif diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 0f8cf16ae992..442d03da3a54 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -857,7 +857,8 @@ static long __kvm_gmem_populate(struct kvm *kvm, struct kvm_memory_slot *slot, return ret; } -long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages, +long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, + long npages, bool may_writeback_src, kvm_gmem_populate_cb post_populate, void *opaque) { struct kvm_memory_slot *slot; @@ -890,8 +891,9 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long if (src) { unsigned long uaddr = (unsigned long)src + i * PAGE_SIZE; + unsigned int flags = may_writeback_src ? FOLL_WRITE : 0; - ret = get_user_pages_fast(uaddr, 1, 0, &src_page); + ret = get_user_pages_fast(uaddr, 1, flags, &src_page); if (ret < 0) break; if (ret != 1) { -- Gitee From c2a4d63b78de18ef79d7d0e7491e3181209a7cf5 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Fri, 22 May 2026 15:46:09 -0700 Subject: [PATCH 68/78] KVM: SEV: Unmap local kmaps in LIFO order, per highmem requirements ANBZ: #40036 commit 138f5f9cbe374ff87084f1d32181d2a4a2d924e8 upstream. Per highmem.h, local kernel mappings must be unmapped in the reserve order they were acquired, following a LIFO (last-in, first-out) stack-based approach, and that failure to do so "is invalid and causes malfunction". Swap the kunmap_local() calls in SNP post-populate flow to ensure the mappings are released in the correct order. Note, because SNP is 64-bit only, the bugs are benign as there are no highmem mappings to unwind. Intel-SIG: commit 138f5f9cbe37 KVM: SEV: Unmap local kmaps in LIFO order, per highmem requirements Fixes: 2a62345b3052 ("KVM: guest_memfd: GUP source pages prior to populating guest memory") Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260522-fix-sev-gmem-post-populate-v2-4-3f196bfad5a1@google.com [sean: call out that the bug is benign] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 00278e3a36ee..40451052de17 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2398,8 +2398,8 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, memcpy(dst_vaddr, src_vaddr, PAGE_SIZE); - kunmap_local(src_vaddr); kunmap_local(dst_vaddr); + kunmap_local(src_vaddr); } ret = rmp_make_private(pfn, gfn << PAGE_SHIFT, PG_LEVEL_4K, @@ -2434,8 +2434,8 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, memcpy(src_vaddr, dst_vaddr, PAGE_SIZE); - kunmap_local(src_vaddr); kunmap_local(dst_vaddr); + kunmap_local(src_vaddr); } out: -- Gitee From 6e9f8609fad484ae4e5b8d9f0f356dbf27bcc0cb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 14:37:11 -0700 Subject: [PATCH 69/78] KVM: TDX: Return EINVAL, not EOPNOTSUPP, for NULL INIT_MEM_REGION source ANBZ: #40036 commit eb606a24386389d6f707a872f2e899cf5875c2c5 upstream. Return EINVAL instead of EOPNOTSUPP if userspace attempts to pass a NULL pointer for the source page of INIT_MEM_REGION, so that KVM's ABI is consistent between TDX and SNP (for LAUNCH_UPDATE). EOPNOTSUPP was chosen to be a forward-looking error code for when guest_memfd supports in-place conversion, but even when in-place conversion comes along, it's an awkward error code as KVM is deliberately choosing to disallow virtual address '0', which is technically a legal userspace address. I.e. it's not so much a lack of support as it is that KVM reserves address '0' to simplify KVM's internal implementation. Opportunistically move the check so that it's co-located with the other checks on the userspace address, and so that it's more obvious that a NULL source address is explicitly disallowed. Intel-SIG: commit eb606a243863 KVM: TDX: Return EINVAL, not EOPNOTSUPP, for NULL INIT_MEM_REGION source Fixes: 2a62345b3052 ("KVM: guest_memfd: GUP source pages prior to populating guest memory") Cc: Yan Zhao Cc: Ackerley Tng Reviewed-by: Xiaoyao Li Acked-by: Kiryl Shutsemau (Meta) Reviewed-by: Binbin Wu Reviewed-by: Yan Zhao Tested-by: Yan Zhao Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260630213711.479692-3-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/vmx/tdx.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index 9979355024e2..51e4e1b0ce15 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3130,9 +3130,6 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - if (!src_page) - return -EOPNOTSUPP; - kvm_tdx->page_add_src = src_page; ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3179,8 +3176,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !PAGE_ALIGNED(region.gpa) || - !region.nr_pages || + if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages || region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) -- Gitee From 552b85f59cac12f3f3a87799b12cadea824c409a Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Fri, 22 May 2026 15:46:10 -0700 Subject: [PATCH 70/78] KVM: SEV: Mark source page dirty when writing back CPUID data on failure ANBZ: #40036 commit 97cd21d57e9bd2da79845178d9250cfd19289cd4 upstream. When writing back CPUID data (provided by trusted firmware) to the source page on failure, mark the page/folio as dirty so that the data isn't lost in the unlikely scenario the page is reclaimed before its read by userspace. Intel-SIG: commit 97cd21d57e9b KVM: SEV: Mark source page dirty when writing back CPUID data on failure Fixes: 2a62345b3052 ("KVM: guest_memfd: GUP source pages prior to populating guest memory") Signed-off-by: Ackerley Tng Link: https://patch.msgid.link/20260522-fix-sev-gmem-post-populate-v2-5-3f196bfad5a1@google.com [sean: use set_page_dirty(), massage changelog] Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/kvm/svm/sev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 40451052de17..2324f5f07cef 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2433,6 +2433,7 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, void *dst_vaddr = kmap_local_pfn(pfn); memcpy(src_vaddr, dst_vaddr, PAGE_SIZE); + set_page_dirty(src_page); kunmap_local(dst_vaddr); kunmap_local(src_vaddr); -- Gitee From 74c398c1a97c6027e8a5482559dd73a29998aab0 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 1 Jun 2025 22:31:29 +0900 Subject: [PATCH 71/78] scripts/misc-check: check missing #include when W=1 ANBZ: #40036 commit a934a57a42f64a40705202f84144b1a29b29f910 upstream. The problem was described in commit 5b20755b7780 ("init: move THIS_MODULE from to "). To summarize it again here: is included by most C files, even though only some of them actually export symbols. This is because some headers, such as include/linux/{module.h,linkage}, needlessly include . I have added a more detailed explanation in the comments of scripts/misc-check. This problem will be fixed in two steps: 1. Add #include directly to C files that use EXPORT_SYMBOL() 2. Remove #include from header files that do not use EXPORT_SYMBOL() This commit addresses step 1; scripts/misc-check will warn about *.[ch] files that use EXPORT_SYMBOL() but do not include . This check is only triggered when the kernel is built with W=1. We need to fix 4000+ files. I hope others will help with this effort. Intel-SIG: commit a934a57a42f6 scripts/misc-check: check missing #include when W=1 Signed-off-by: Masahiro Yamada [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- scripts/misc-check | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/scripts/misc-check b/scripts/misc-check index d40d5484e0c5..eeeff5d8f600 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -16,4 +16,47 @@ check_tracked_ignored_files () { sed 's/$/: warning: ignored by one of the .gitignore files/' >&2 } +# Check for missing #include +# +# The rule for including is very simple: +# Include only when you use EXPORT_SYMBOL(). That's it. +# +# However, some headers include even though they are completely +# unrelated to EXPORT_SYMBOL(). +# +# One example is include/linux/module.h. Please note and +# are orthogonal. should be included by files +# that can be compiled as modules. In other words, should be +# included by EXPORT_SYMBOL consumers. In contrast, should be +# included from EXPORT_SYMBOL providers, which may or may not be modular. +# Hence, include/linux/module.h should *not* include . +# +# Another example is include/linux/linkage.h, which is completely unrelated to +# EXPORT_SYMBOL(). Worse, it is included by most C files, which means, most C +# files end up including , even though only some of them +# actually export symbols. Hence, include/linux/linkage.h should *not* include +# . +# +# Before fixing such headers, we must ensure that C files using EXPORT_SYMBOL() +# include directly, since many C files currently rely on +# being included indirectly (likely, via etc.). +# +# Therefore, this check. +# +# The problem is simple - the warned files use EXPORT_SYMBOL(), but do not +# include . Please add #include to them. +# +# If the included headers are sorted alphabetically, please insert +# in the appropriate position to maintain the sort order. +# For this reason, this script only checks missing , but +# does not automatically fix it. +check_missing_include_linux_export_h () { + + git -C "${srctree:-.}" grep --files-with-matches -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' \ + -- '*.[ch]' :^tools/ :^include/linux/export.h | + xargs -r git -C "${srctree:-.}" grep --files-without-match '#include[[:space:]]*' | + xargs -r printf "%s: warning: EXPORT_SYMBOL() is used, but #include is missing\n" >&2 +} + check_tracked_ignored_files +check_missing_include_linux_export_h -- Gitee From b6fcb8dbce517400cb4cf3a90f914e34b429e8ce Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 1 Jun 2025 22:31:30 +0900 Subject: [PATCH 72/78] scripts/misc-check: check unnecessary #include when W=1 ANBZ: #40036 commit 7d95680d64ac8e836c35fd56efe77eac4e9cc26b upstream. Another issue with is that it is sometimes included even when EXPORT_SYMBOL() is not used at all. Some headers (e.g. include/linux/linkage.h>) cannot be fixed for now for the reason described in the previous commit. This commit adds a warning for *.c files that include but do not use EXPORT_SYMBOL() when the kernel is built with W=1. Intel-SIG: commit 7d95680d64ac scripts/misc-check: check unnecessary #include when W=1 Signed-off-by: Masahiro Yamada [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- scripts/misc-check | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/misc-check b/scripts/misc-check index eeeff5d8f600..f6e33cc93df7 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -58,5 +58,17 @@ check_missing_include_linux_export_h () { xargs -r printf "%s: warning: EXPORT_SYMBOL() is used, but #include is missing\n" >&2 } +# If you do not use EXPORT_SYMBOL(), please do not include . +# Currently, this is checked for *.c files, but not for *.h files, because some +# *.c files rely on being included indirectly. +check_unnecessary_include_linux_export_h () { + + git -C "${srctree:-.}" grep --files-with-matches '#include[[:space:]]*' \ + -- '*.[c]' :^tools/ | + xargs -r git -C "${srctree:-.}" grep --files-without-match -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' | + xargs -r printf "%s: warning: EXPORT_SYMBOL() is not used, but #include is present\n" >&2 +} + check_tracked_ignored_files check_missing_include_linux_export_h +check_unnecessary_include_linux_export_h -- Gitee From fad6fae28ec09427bd16a99cb2957460359b258d Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 25 Aug 2025 17:00:37 +0200 Subject: [PATCH 73/78] scripts/misc-check: update export checks for EXPORT_SYMBOL_FOR_MODULES() ANBZ: #40036 commit 0354e81b7bd629f9c3379c9524e988ebc504fa25 upstream. The module export checks are looking for EXPORT_SYMBOL_GPL_FOR_MODULES() which was renamed to EXPORT_SYMBOL_FOR_MODULES(). Update the checks. Intel-SIG: commit 0354e81b7bd6 scripts/misc-check: update export checks for EXPORT_SYMBOL_FOR_MODULES() Fixes: 6d3c3ca4c77e ("module: Rename EXPORT_SYMBOL_GPL_FOR_MODULES to EXPORT_SYMBOL_FOR_MODULES") Signed-off-by: Vlastimil Babka Reviewed-by: Daniel Gomez Reviewed-by: Nicolas Schier Link: https://lore.kernel.org/r/20250825-export_modules_fix-v1-1-5c331e949538@suse.cz Signed-off-by: Nathan Chancellor [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- scripts/misc-check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/misc-check b/scripts/misc-check index f6e33cc93df7..df7cad68fde4 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -52,7 +52,7 @@ check_tracked_ignored_files () { # does not automatically fix it. check_missing_include_linux_export_h () { - git -C "${srctree:-.}" grep --files-with-matches -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' \ + git -C "${srctree:-.}" grep --files-with-matches -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_FOR_MODULES)\(.*\)' \ -- '*.[ch]' :^tools/ :^include/linux/export.h | xargs -r git -C "${srctree:-.}" grep --files-without-match '#include[[:space:]]*' | xargs -r printf "%s: warning: EXPORT_SYMBOL() is used, but #include is missing\n" >&2 @@ -65,7 +65,7 @@ check_unnecessary_include_linux_export_h () { git -C "${srctree:-.}" grep --files-with-matches '#include[[:space:]]*' \ -- '*.[c]' :^tools/ | - xargs -r git -C "${srctree:-.}" grep --files-without-match -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_GPL_FOR_MODULES)\(.*\)' | + xargs -r git -C "${srctree:-.}" grep --files-without-match -E 'EXPORT_SYMBOL((_NS)?(_GPL)?|_FOR_MODULES)\(.*\)' | xargs -r printf "%s: warning: EXPORT_SYMBOL() is not used, but #include is present\n" >&2 } -- Gitee From 416a9ae4f39a7b4cbd2729f7e6a6f5d6a6b7b322 Mon Sep 17 00:00:00 2001 From: Mikhail Gavrilov Date: Tue, 5 May 2026 04:54:35 +0500 Subject: [PATCH 74/78] x86/virt: Silence RCU lockdep splat in emergency virt callback path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ANBZ: #40036 commit fff82ea9d900b6bbebc58d34b7a63789de1ad10d upstream. x86_virt_invoke_kvm_emergency_callback() reaches rcu_dereference() through machine_crash_shutdown() with IRQs disabled but with RCU not necessarily watching the crashing CPU, which triggers a suspicious RCU usage splat on debug kernels (CONFIG_PROVE_RCU=y) during panic/kdump: WARNING: suspicious RCU usage arch/x86/virt/hw.c:52 suspicious rcu_dereference_check() usage! rcu_scheduler_active = 2, debug_locks = 1 1 lock held by tee/11119: #0: ffff8881fa32c440 (sb_writers#3){.+.+}-{0:0}, at: ksys_write Call Trace: dump_stack_lvl+0x84/0xd0 lockdep_rcu_suspicious.cold+0x37/0x8f x86_virt_invoke_kvm_emergency_callback+0x5f/0x70 x86_svm_emergency_disable_virtualization_cpu+0x2a/0x30 x86_virt_emergency_disable_virtualization_cpu+0x6b/0x90 native_machine_crash_shutdown+0x72/0x170 __crash_kexec+0x137/0x280 panic+0xce/0xd0 sysrq_handle_crash+0x1f/0x20 __handle_sysrq.cold+0x192/0x335 write_sysrq_trigger+0x8c/0xc0 proc_reg_write+0x1c3/0x3c0 vfs_write+0x1d0/0xf80 ksys_write+0x116/0x250 do_syscall_64+0x11c/0x1480 entry_SYSCALL_64_after_hwframe+0x76/0x7e A truly correct fix is non-trivial: the RCU usage genuinely is wrong in panic context (RCU may ignore the crashing CPU during synchronization), and a concurrent KVM module unload could in principle race with the callback read; see commit 2baa33a8ddd6 ("KVM: x86: Leave user-return notifier registered on reboot/shutdown") which notes that nothing prevents module unload during panic/reboot. However, the alternatives are worse: - smp_store_release()/smp_load_acquire() handles ordering but not liveness; the kernel still needs to keep the module text alive while the callback is in flight. - Taking a lock in the panic path is risky — any lock could be held by a CPU that has already been NMI'd to a halt. Use rcu_dereference_raw() to silence the splat and accept the vanishingly small remaining race. Panic context inherently cannot guarantee complete correctness; the goal here is to keep debug builds quiet on the kdump path so the splat doesn't obscure the actual kernel state being captured. Reproducible on a debug kernel (CONFIG_PROVE_LOCKING=y, CONFIG_PROVE_RCU=y) with kvm_amd or kvm_intel loaded by triggering kdump: echo c > /proc/sysrq-trigger Intel-SIG: commit fff82ea9d900 x86/virt: Silence RCU lockdep splat in emergency virt callback path Suggested-by: Sean Christopherson Fixes: 428afac5a8ea ("KVM: x86: Move bulk of emergency virtualizaton logic to virt subsystem") Signed-off-by: Mikhail Gavrilov Acked-by: Sean Christopherson Link: https://patch.msgid.link/20260504235435.90957-1-mikhail.v.gavrilov@gmail.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/hw.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 6b7ff8f85000..7e0131416b7c 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -49,7 +49,20 @@ static void x86_virt_invoke_kvm_emergency_callback(void) { cpu_emergency_virt_cb *kvm_callback; - kvm_callback = rcu_dereference(kvm_emergency_callback); + /* + * RCU may not be watching the crashing CPU here, so rcu_dereference() + * triggers a suspicious-RCU-usage splat. In principle, a concurrent + * KVM module unload could race with this read; see commit 2baa33a8ddd6 + * ("KVM: x86: Leave user-return notifier registered on reboot/shutdown") + * which notes that nothing prevents module unload during panic/reboot. + * + * However, taking a lock here would be riskier than the current race: + * the system is going down via NMI shootdown, and any lock could be + * held by an already-stopped CPU. Use rcu_dereference_raw() to silence + * the lockdep splat and accept the comically small remaining race; + * panic context inherently cannot guarantee complete correctness. + */ + kvm_callback = rcu_dereference_raw(kvm_emergency_callback); if (kvm_callback) kvm_callback(); } -- Gitee From 29152268003f200d127b6d07e3535bd467694202 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Apr 2026 12:13:41 -0700 Subject: [PATCH 75/78] x86/virt: Treat SVM as unsupported when running as an SEV+ guest ANBZ: #40036 commit e30aa03d032df0f3ee5efb1995a7a2fe662177be upstream. When running as an SEV+ guest, treat SVM as unsupported even if CPUID (and other reporting, e.g. MSRs) enumerate support for SVM, as KVM doesn't support nested virtualization within an SEV VM (KVM would need to explicitly share all VMCBs and other assets with the untrusted host), let alone running nested VMs within SEV-ES+ guests (e.g. emulating VMLOAD, VMSAVE, and VMRUN all require access to guest register state). And outside of KVM, there is no in-tree user of SVM enabling. Arguably, the hypervisor/VMM (e.g. QEMU) should clear SVM from guest CPUID for SEV VMs, especially for SEV-ES+, but super duper technically, it's feasible to run nested VMs in SEV+ guests (with many caveats). More importantly, Linux-as-a-guest has played nice with SVM being advertised to SEV+ guests for a long time. Treating SVM as unsupported fixes a regression where a clean shutdown of an SEV-ES+ guest degrades into an abrupt termination. Due to a gnarly virtualization hole in SEV-ES (the architecture), where EFER must NOT be intercepted by the hypervisor (because the untrusted hypervisor can't set e.g. EFER.LME on behalf o the guest), the _host's_ EFER.SVME is visible to the guest. Because EFER.SVME must be always '1' while in guest mode, Linux-the-guest sees EFER.SVME=1 even when _its_ EFER.SVME is '0', thinks it has enabled virtualization, and ultimately can cause x86_svm_emergency_disable_virtualization_cpu() to execute STGI to ensure GIF is enabled. Executing STGI _should_ be fine, except Linux is a also wee bit paranoid when running as an SEV-ES guest. Because L0 sees EFER.SVME=0 for the guest, a well-behaved L0 hypervisor will intercept STGI (to inject #UD), and thus generate a #VC on the STGI. Which, again, should be fine. Unfortunately, vc_check_opcode_bytes() fails to account for STGI and other SVM instructions, throws a fatal error, and triggers a termination request. In a perfect world, the #VC handler would be more forgiving of unknown intercepts, especially when the #VC happened on an instruction with exception fixup. For now, just fix the immediate regression. Intel-SIG: commit e30aa03d032d x86/virt: Treat SVM as unsupported when running as an SEV+ guest Fixes: 428afac5a8ea ("KVM: x86: Move bulk of emergency virtualizaton logic to virt subsystem") Reported-by: Srikanth Aithal Closes: https://lore.kernel.org/all/c820e242-9f3a-4210-b414-19d11b022404@amd.com Link: https://patch.msgid.link/20260409191341.1932853-1-seanjc@google.com Signed-off-by: Sean Christopherson [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- arch/x86/virt/hw.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c index 7e0131416b7c..9bc78b517b1f 100644 --- a/arch/x86/virt/hw.c +++ b/arch/x86/virt/hw.c @@ -285,7 +285,8 @@ static __init int x86_svm_init(void) .emergency_disable_virtualization_cpu = x86_svm_emergency_disable_virtualization_cpu, }; - if (!cpu_feature_enabled(X86_FEATURE_SVM)) + if (!cpu_feature_enabled(X86_FEATURE_SVM) || + cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) return -EOPNOTSUPP; memcpy(&virt_ops, &svm_ops, sizeof(virt_ops)); -- Gitee From 4cefa37098c0aa80369d7f0852b02af9b51b58ae Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Sun, 1 Jun 2025 22:31:27 +0900 Subject: [PATCH 76/78] kbuild: move W=1 check for scripts/misc-check to top-level Makefile ANBZ: #40036 commit 89e7fecf5ce2e85a323e58f09aa808218a37079a upstream. This script is executed only when ${KBUILD_EXTRA_WARN} contains 1. Move this check to the top-level Makefile to allow more checks to be easily added to this script. Intel-SIG: commit 89e7fecf5ce2 kbuild: move W=1 check for scripts/misc-check to top-level Makefile Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Makefile | 3 +++ scripts/misc-check | 9 +-------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index adedd2611205..2710af784d5a 100644 --- a/Makefile +++ b/Makefile @@ -1793,9 +1793,12 @@ rustfmtcheck: rustfmt # Misc # --------------------------------------------------------------------------- +# Run misc checks when ${KBUILD_EXTRA_WARN} contains 1 PHONY += misc-check +ifneq ($(findstring 1,$(KBUILD_EXTRA_WARN)),) misc-check: $(Q)$(srctree)/scripts/misc-check +endif all: misc-check diff --git a/scripts/misc-check b/scripts/misc-check index df7cad68fde4..13642f08d609 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -3,15 +3,8 @@ set -e -# Detect files that are tracked but ignored by git. This is checked only when -# ${KBUILD_EXTRA_WARN} contains 1, git is installed, and the source tree is -# tracked by git. +# Detect files that are tracked but ignored by git. check_tracked_ignored_files () { - case "${KBUILD_EXTRA_WARN}" in - *1*) ;; - *) return;; - esac - git -C ${srctree:-.} ls-files -i -c --exclude-per-directory=.gitignore 2>/dev/null | sed 's/$/: warning: ignored by one of the .gitignore files/' >&2 } -- Gitee From f64adf318b3690c312bda2e29112bab29d5066ac Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 13 Jun 2025 01:08:48 +0900 Subject: [PATCH 77/78] kbuild: move warnings about linux/export.h from W=1 to W=2 ANBZ: #40036 commit a6a7946bd691940cfe7289ae6dfb1f077516df72 upstream. This hides excessive warnings, as nobody builds with W=2. Intel-SIG: commit a6a7946bd691 kbuild: move warnings about linux/export.h from W=1 to W=2 Fixes: a934a57a42f6 ("scripts/misc-check: check missing #include when W=1") Fixes: 7d95680d64ac ("scripts/misc-check: check unnecessary #include when W=1") Signed-off-by: Masahiro Yamada Acked-by: Arnd Bergmann Reviewed-by: Nathan Chancellor Acked-by: Heiko Carstens [ Jun Miao: amend commit log ] Signed-off-by: Jun Miao --- Makefile | 3 --- scripts/misc-check | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 2710af784d5a..adedd2611205 100644 --- a/Makefile +++ b/Makefile @@ -1793,12 +1793,9 @@ rustfmtcheck: rustfmt # Misc # --------------------------------------------------------------------------- -# Run misc checks when ${KBUILD_EXTRA_WARN} contains 1 PHONY += misc-check -ifneq ($(findstring 1,$(KBUILD_EXTRA_WARN)),) misc-check: $(Q)$(srctree)/scripts/misc-check -endif all: misc-check diff --git a/scripts/misc-check b/scripts/misc-check index 13642f08d609..f88624a28cc2 100755 --- a/scripts/misc-check +++ b/scripts/misc-check @@ -62,6 +62,15 @@ check_unnecessary_include_linux_export_h () { xargs -r printf "%s: warning: EXPORT_SYMBOL() is not used, but #include is present\n" >&2 } -check_tracked_ignored_files -check_missing_include_linux_export_h -check_unnecessary_include_linux_export_h +case "${KBUILD_EXTRA_WARN}" in +*1*) + check_tracked_ignored_files + ;; +esac + +case "${KBUILD_EXTRA_WARN}" in +*2*) + check_missing_include_linux_export_h + check_unnecessary_include_linux_export_h + ;; +esac -- Gitee From 1c44f7712d7af9a4c07f6c2b522eeb56f85e86b6 Mon Sep 17 00:00:00 2001 From: Jun Miao Date: Wed, 9 Sep 2026 06:02:01 -0400 Subject: [PATCH 78/78] Revert "KVM: guest_memfd: Use AS_INACCESSIBLE when creating guest_memfd inode" ANBZ: #40036 This reverts commit 8fe580e41a4f7531e5eed0f08fbd28b7e914029e. Because this patch was mistakenly added previously, and the follow patch can fix it. For more detailed reasons, please refer to the information on the merged patch below: Subject: commit 76abdb8d0830 "mm, virt: merge AS_UNMOVABLE and AS_INACCESSIBLE" ANBZ: #34453 commit 27e6a24a4cf3d25421c0f6ebb7c39f45fc14d20f upstream. --- virt/kvm/guest_memfd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 442d03da3a54..3ad3e7a0df78 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -589,7 +589,6 @@ static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags) inode->i_op = &kvm_gmem_iops; inode->i_mapping->a_ops = &kvm_gmem_aops; - inode->i_mapping->flags |= AS_INACCESSIBLE; inode->i_mode |= S_IFREG; inode->i_size = size; mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); -- Gitee