Commit 55c7401d authored by Marc Zyngier's avatar Marc Zyngier
Browse files

arm64: KVM: HYP mode world switch implementation



The HYP mode world switch in all its glory.

Implements save/restore of host/guest registers, EL2 trapping,
IPA resolution, and additional services (tlb invalidation).
Reviewed-by: default avatarChristopher Covington <cov@codeaurora.org>
Signed-off-by: default avatarMarc Zyngier <marc.zyngier@arm.com>
parent 092bd143
......@@ -104,5 +104,38 @@ int main(void)
BLANK();
DEFINE(TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
DEFINE(TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
BLANK();
#ifdef CONFIG_KVM_ARM_HOST
DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
DEFINE(CPU_GP_REGS, offsetof(struct kvm_cpu_context, gp_regs));
DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_regs, regs));
DEFINE(CPU_FP_REGS, offsetof(struct kvm_regs, fp_regs));
DEFINE(CPU_SP_EL1, offsetof(struct kvm_regs, sp_el1));
DEFINE(CPU_ELR_EL1, offsetof(struct kvm_regs, elr_el1));
DEFINE(CPU_SPSR, offsetof(struct kvm_regs, spsr));
DEFINE(CPU_SYSREGS, offsetof(struct kvm_cpu_context, sys_regs));
DEFINE(VCPU_ESR_EL2, offsetof(struct kvm_vcpu, arch.fault.esr_el2));
DEFINE(VCPU_FAR_EL2, offsetof(struct kvm_vcpu, arch.fault.far_el2));
DEFINE(VCPU_HPFAR_EL2, offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2));
DEFINE(VCPU_IRQ_LINES, offsetof(struct kvm_vcpu, arch.irq_lines));
DEFINE(VCPU_HOST_CONTEXT, offsetof(struct kvm_vcpu, arch.host_cpu_context));
DEFINE(VCPU_TIMER_CNTV_CTL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
DEFINE(VCPU_TIMER_CNTV_CVAL, offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_cval));
DEFINE(KVM_TIMER_CNTVOFF, offsetof(struct kvm, arch.timer.cntvoff));
DEFINE(KVM_TIMER_ENABLED, offsetof(struct kvm, arch.timer.enabled));
DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm));
DEFINE(VCPU_VGIC_CPU, offsetof(struct kvm_vcpu, arch.vgic_cpu));
DEFINE(VGIC_CPU_HCR, offsetof(struct vgic_cpu, vgic_hcr));
DEFINE(VGIC_CPU_VMCR, offsetof(struct vgic_cpu, vgic_vmcr));
DEFINE(VGIC_CPU_MISR, offsetof(struct vgic_cpu, vgic_misr));
DEFINE(VGIC_CPU_EISR, offsetof(struct vgic_cpu, vgic_eisr));
DEFINE(VGIC_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_elrsr));
DEFINE(VGIC_CPU_APR, offsetof(struct vgic_cpu, vgic_apr));
DEFINE(VGIC_CPU_LR, offsetof(struct vgic_cpu, vgic_lr));
DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
#endif
return 0;
}
/*
* Copyright (C) 2012,2013 - ARM Ltd
* Author: Marc Zyngier <marc.zyngier@arm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/linkage.h>
#include <linux/irqchip/arm-gic.h>
#include <asm/assembler.h>
#include <asm/memory.h>
#include <asm/asm-offsets.h>
#include <asm/fpsimdmacros.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_mmu.h>
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
#define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
#define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
.text
.pushsection .hyp.text, "ax"
.align PAGE_SHIFT
__kvm_hyp_code_start:
.globl __kvm_hyp_code_start
.macro save_common_regs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_XREG_OFFSET(19)
stp x19, x20, [x3]
stp x21, x22, [x3, #16]
stp x23, x24, [x3, #32]
stp x25, x26, [x3, #48]
stp x27, x28, [x3, #64]
stp x29, lr, [x3, #80]
mrs x19, sp_el0
mrs x20, elr_el2 // EL1 PC
mrs x21, spsr_el2 // EL1 pstate
stp x19, x20, [x3, #96]
str x21, [x3, #112]
mrs x22, sp_el1
mrs x23, elr_el1
mrs x24, spsr_el1
str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
.endm
.macro restore_common_regs
// x2: base address for cpu context
// x3: tmp register
ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
msr sp_el1, x22
msr elr_el1, x23
msr spsr_el1, x24
add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
ldp x19, x20, [x3]
ldr x21, [x3, #16]
msr sp_el0, x19
msr elr_el2, x20 // EL1 PC
msr spsr_el2, x21 // EL1 pstate
add x3, x2, #CPU_XREG_OFFSET(19)
ldp x19, x20, [x3]
ldp x21, x22, [x3, #16]
ldp x23, x24, [x3, #32]
ldp x25, x26, [x3, #48]
ldp x27, x28, [x3, #64]
ldp x29, lr, [x3, #80]
.endm
.macro save_host_regs
save_common_regs
.endm
.macro restore_host_regs
restore_common_regs
.endm
.macro save_fpsimd
// x2: cpu context address
// x3, x4: tmp regs
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
fpsimd_save x3, 4
.endm
.macro restore_fpsimd
// x2: cpu context address
// x3, x4: tmp regs
add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
fpsimd_restore x3, 4
.endm
.macro save_guest_regs
// x0 is the vcpu address
// x1 is the return code, do not corrupt!
// x2 is the cpu context
// x3 is a tmp register
// Guest's x0-x3 are on the stack
// Compute base to save registers
add x3, x2, #CPU_XREG_OFFSET(4)
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
stp x8, x9, [x3, #32]
stp x10, x11, [x3, #48]
stp x12, x13, [x3, #64]
stp x14, x15, [x3, #80]
stp x16, x17, [x3, #96]
str x18, [x3, #112]
pop x6, x7 // x2, x3
pop x4, x5 // x0, x1
add x3, x2, #CPU_XREG_OFFSET(0)
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
save_common_regs
.endm
.macro restore_guest_regs
// x0 is the vcpu address.
// x2 is the cpu context
// x3 is a tmp register
// Prepare x0-x3 for later restore
add x3, x2, #CPU_XREG_OFFSET(0)
ldp x4, x5, [x3]
ldp x6, x7, [x3, #16]
push x4, x5 // Push x0-x3 on the stack
push x6, x7
// x4-x18
ldp x4, x5, [x3, #32]
ldp x6, x7, [x3, #48]
ldp x8, x9, [x3, #64]
ldp x10, x11, [x3, #80]
ldp x12, x13, [x3, #96]
ldp x14, x15, [x3, #112]
ldp x16, x17, [x3, #128]
ldr x18, [x3, #144]
// x19-x29, lr, sp*, elr*, spsr*
restore_common_regs
// Last bits of the 64bit state
pop x2, x3
pop x0, x1
// Do not touch any register after this!
.endm
/*
* Macros to perform system register save/restore.
*
* Ordering here is absolutely critical, and must be kept consistent
* in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
* and in kvm_asm.h.
*
* In other words, don't touch any of these unless you know what
* you are doing.
*/
.macro save_sysregs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
mrs x4, vmpidr_el2
mrs x5, csselr_el1
mrs x6, sctlr_el1
mrs x7, actlr_el1
mrs x8, cpacr_el1
mrs x9, ttbr0_el1
mrs x10, ttbr1_el1
mrs x11, tcr_el1
mrs x12, esr_el1
mrs x13, afsr0_el1
mrs x14, afsr1_el1
mrs x15, far_el1
mrs x16, mair_el1
mrs x17, vbar_el1
mrs x18, contextidr_el1
mrs x19, tpidr_el0
mrs x20, tpidrro_el0
mrs x21, tpidr_el1
mrs x22, amair_el1
mrs x23, cntkctl_el1
stp x4, x5, [x3]
stp x6, x7, [x3, #16]
stp x8, x9, [x3, #32]
stp x10, x11, [x3, #48]
stp x12, x13, [x3, #64]
stp x14, x15, [x3, #80]
stp x16, x17, [x3, #96]
stp x18, x19, [x3, #112]
stp x20, x21, [x3, #128]
stp x22, x23, [x3, #144]
.endm
.macro restore_sysregs
// x2: base address for cpu context
// x3: tmp register
add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
ldp x4, x5, [x3]
ldp x6, x7, [x3, #16]
ldp x8, x9, [x3, #32]
ldp x10, x11, [x3, #48]
ldp x12, x13, [x3, #64]
ldp x14, x15, [x3, #80]
ldp x16, x17, [x3, #96]
ldp x18, x19, [x3, #112]
ldp x20, x21, [x3, #128]
ldp x22, x23, [x3, #144]
msr vmpidr_el2, x4
msr csselr_el1, x5
msr sctlr_el1, x6
msr actlr_el1, x7
msr cpacr_el1, x8
msr ttbr0_el1, x9
msr ttbr1_el1, x10
msr tcr_el1, x11
msr esr_el1, x12
msr afsr0_el1, x13
msr afsr1_el1, x14
msr far_el1, x15
msr mair_el1, x16
msr vbar_el1, x17
msr contextidr_el1, x18
msr tpidr_el0, x19
msr tpidrro_el0, x20
msr tpidr_el1, x21
msr amair_el1, x22
msr cntkctl_el1, x23
.endm
.macro activate_traps
ldr x2, [x0, #VCPU_IRQ_LINES]
ldr x1, [x0, #VCPU_HCR_EL2]
orr x2, x2, x1
msr hcr_el2, x2
ldr x2, =(CPTR_EL2_TTA)
msr cptr_el2, x2
ldr x2, =(1 << 15) // Trap CP15 Cr=15
msr hstr_el2, x2
mrs x2, mdcr_el2
and x2, x2, #MDCR_EL2_HPMN_MASK
orr x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
msr mdcr_el2, x2
.endm
.macro deactivate_traps
mov x2, #HCR_RW
msr hcr_el2, x2
msr cptr_el2, xzr
msr hstr_el2, xzr
mrs x2, mdcr_el2
and x2, x2, #MDCR_EL2_HPMN_MASK
msr mdcr_el2, x2
.endm
.macro activate_vm
ldr x1, [x0, #VCPU_KVM]
kern_hyp_va x1
ldr x2, [x1, #KVM_VTTBR]
msr vttbr_el2, x2
.endm
.macro deactivate_vm
msr vttbr_el2, xzr
.endm
__save_sysregs:
save_sysregs
ret
__restore_sysregs:
restore_sysregs
ret
__save_fpsimd:
save_fpsimd
ret
__restore_fpsimd:
restore_fpsimd
ret
/*
* u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
*
* This is the world switch. The first half of the function
* deals with entering the guest, and anything from __kvm_vcpu_return
* to the end of the function deals with reentering the host.
* On the enter path, only x0 (vcpu pointer) must be preserved until
* the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
* code) must both be preserved until the epilogue.
* In both cases, x2 points to the CPU context we're saving/restoring from/to.
*/
ENTRY(__kvm_vcpu_run)
kern_hyp_va x0
msr tpidr_el2, x0 // Save the vcpu register
// Host context
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
save_host_regs
bl __save_fpsimd
bl __save_sysregs
activate_traps
activate_vm
// Guest context
add x2, x0, #VCPU_CONTEXT
bl __restore_sysregs
bl __restore_fpsimd
restore_guest_regs
// That's it, no more messing around.
eret
__kvm_vcpu_return:
// Assume x0 is the vcpu pointer, x1 the return code
// Guest's x0-x3 are on the stack
// Guest context
add x2, x0, #VCPU_CONTEXT
save_guest_regs
bl __save_fpsimd
bl __save_sysregs
deactivate_traps
deactivate_vm
// Host context
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
bl __restore_sysregs
bl __restore_fpsimd
restore_host_regs
mov x0, x1
ret
END(__kvm_vcpu_run)
// void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
ENTRY(__kvm_tlb_flush_vmid_ipa)
kern_hyp_va x0
ldr x2, [x0, #KVM_VTTBR]
msr vttbr_el2, x2
isb
/*
* We could do so much better if we had the VA as well.
* Instead, we invalidate Stage-2 for this IPA, and the
* whole of Stage-1. Weep...
*/
tlbi ipas2e1is, x1
dsb sy
tlbi vmalle1is
dsb sy
isb
msr vttbr_el2, xzr
ret
ENDPROC(__kvm_tlb_flush_vmid_ipa)
ENTRY(__kvm_flush_vm_context)
tlbi alle1is
ic ialluis
dsb sy
ret
ENDPROC(__kvm_flush_vm_context)
__kvm_hyp_panic:
// Guess the context by looking at VTTBR:
// If zero, then we're already a host.
// Otherwise restore a minimal host context before panicing.
mrs x0, vttbr_el2
cbz x0, 1f
mrs x0, tpidr_el2
deactivate_traps
deactivate_vm
ldr x2, [x0, #VCPU_HOST_CONTEXT]
kern_hyp_va x2
bl __restore_sysregs
1: adr x0, __hyp_panic_str
adr x1, 2f
ldp x2, x3, [x1]
sub x0, x0, x2
add x0, x0, x3
mrs x1, spsr_el2
mrs x2, elr_el2
mrs x3, esr_el2
mrs x4, far_el2
mrs x5, hpfar_el2
mrs x6, par_el1
mrs x7, tpidr_el2
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
PSR_MODE_EL1h)
msr spsr_el2, lr
ldr lr, =panic
msr elr_el2, lr
eret
.align 3
2: .quad HYP_PAGE_OFFSET
.quad PAGE_OFFSET
ENDPROC(__kvm_hyp_panic)
__hyp_panic_str:
.ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0"
.align 2
ENTRY(kvm_call_hyp)
hvc #0
ret
ENDPROC(kvm_call_hyp)
.macro invalid_vector label, target
.align 2
\label:
b \target
ENDPROC(\label)
.endm
/* None of these should ever happen */
invalid_vector el2t_sync_invalid, __kvm_hyp_panic
invalid_vector el2t_irq_invalid, __kvm_hyp_panic
invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
invalid_vector el2t_error_invalid, __kvm_hyp_panic
invalid_vector el2h_sync_invalid, __kvm_hyp_panic
invalid_vector el2h_irq_invalid, __kvm_hyp_panic
invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
invalid_vector el2h_error_invalid, __kvm_hyp_panic
invalid_vector el1_sync_invalid, __kvm_hyp_panic
invalid_vector el1_irq_invalid, __kvm_hyp_panic
invalid_vector el1_fiq_invalid, __kvm_hyp_panic
invalid_vector el1_error_invalid, __kvm_hyp_panic
el1_sync: // Guest trapped into EL2
push x0, x1
push x2, x3
mrs x1, esr_el2
lsr x2, x1, #ESR_EL2_EC_SHIFT
cmp x2, #ESR_EL2_EC_HVC64
b.ne el1_trap
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
cbnz x3, el1_trap // called HVC
/* Here, we're pretty sure the host called HVC. */
pop x2, x3
pop x0, x1
push lr, xzr
/*
* Compute the function address in EL2, and shuffle the parameters.
*/
kern_hyp_va x0
mov lr, x0
mov x0, x1
mov x1, x2
mov x2, x3
blr lr
pop lr, xzr
eret
el1_trap:
/*
* x1: ESR
* x2: ESR_EC
*/
cmp x2, #ESR_EL2_EC_DABT
mov x0, #ESR_EL2_EC_IABT
ccmp x2, x0, #4, ne
b.ne 1f // Not an abort we care about
/* This is an abort. Check for permission fault */
and x2, x1, #ESR_EL2_FSC_TYPE
cmp x2, #FSC_PERM
b.ne 1f // Not a permission fault
/*
* Check for Stage-1 page table walk, which is guaranteed
* to give a valid HPFAR_EL2.
*/
tbnz x1, #7, 1f // S1PTW is set
/*
* Permission fault, HPFAR_EL2 is invalid.
* Resolve the IPA the hard way using the guest VA.
* Stage-1 translation already validated the memory access rights.
* As such, we can use the EL1 translation regime, and don't have
* to distinguish between EL0 and EL1 access.
*/
mrs x2, far_el2
at s1e1r, x2
isb
/* Read result */
mrs x3, par_el1
tbnz x3, #0, 3f // Bail out if we failed the translation
ubfx x3, x3, #12, #36 // Extract IPA
lsl x3, x3, #4 // and present it like HPFAR
b 2f
1: mrs x3, hpfar_el2
mrs x2, far_el2
2: mrs x0, tpidr_el2
str x1, [x0, #VCPU_ESR_EL2]
str x2, [x0, #VCPU_FAR_EL2]
str x3, [x0, #VCPU_HPFAR_EL2]
mov x1, #ARM_EXCEPTION_TRAP
b __kvm_vcpu_return
/*
* Translation failed. Just return to the guest and
* let it fault again. Another CPU is probably playing
* behind our back.
*/