/* * Asm versions of Xen pv-ops, suitable for either direct use or * inlining. The inline versions are the same as the direct-use * versions, with the pre- and post-amble chopped off. * * This code is encoded for size rather than absolute efficiency, with * a view to being able to inline as much as possible. * * We only bother with direct forms (ie, vcpu in pda) of the * operations here; the indirect forms are better handled in C, since * they're generally too large to inline anyway. */ #include <asm/errno.h> #include <asm/percpu.h> #include <asm/processor-flags.h> #include <asm/segment.h> #include <xen/interface/xen.h> #include "xen-asm.h" ENTRY(xen_adjust_exception_frame) mov 8+0(%rsp), %rcx mov 8+8(%rsp), %r11 ret $16 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32 /* * Xen64 iret frame: * * ss * rsp * rflags * cs * rip <-- standard iret frame * * flags * * rcx } * r11 }<-- pushed by hypercall page * rsp->rax } */ ENTRY(xen_iret) pushq $0 1: jmp hypercall_iret ENDPATCH(xen_iret) RELOC(xen_iret, 1b+1) /* * sysexit is not used for 64-bit processes, so it's only ever used to * return to 32-bit compat userspace. */ ENTRY(xen_sysexit) pushq $__USER32_DS pushq %rcx pushq $X86_EFLAGS_IF pushq $__USER32_CS pushq %rdx pushq $0 1: jmp hypercall_iret ENDPATCH(xen_sysexit) RELOC(xen_sysexit, 1b+1) ENTRY(xen_sysret64) /* * We're already on the usermode stack at this point, but * still with the kernel gs, so we can easily switch back */ movq %rsp, PER_CPU_VAR(old_rsp) movq PER_CPU_VAR(kernel_stack), %rsp pushq $__USER_DS pushq PER_CPU_VAR(old_rsp) pushq %r11 pushq $__USER_CS pushq %rcx pushq $VGCF_in_syscall 1: jmp hypercall_iret ENDPATCH(xen_sysret64) RELOC(xen_sysret64, 1b+1) ENTRY(xen_sysret32) /* * We're already on the usermode stack at this point, but * still with the kernel gs, so we can easily switch back */ movq %rsp, PER_CPU_VAR(old_rsp) movq PER_CPU_VAR(kernel_stack), %rsp pushq $__USER32_DS pushq PER_CPU_VAR(old_rsp) pushq %r11 pushq $__USER32_CS pushq %rcx pushq $VGCF_in_syscall 1: jmp hypercall_iret ENDPATCH(xen_sysret32) RELOC(xen_sysret32, 1b+1) /* * Xen handles syscall callbacks much like ordinary exceptions, which * means we have: * - kernel gs * - kernel rsp * - an iret-like stack frame on the stack (including rcx and r11): * ss * rsp * rflags * cs * rip * r11 * rsp->rcx * * In all the entrypoints, we undo all that to make it look like a * CPU-generated syscall/sysenter and jump to the normal entrypoint. */ .macro undo_xen_syscall mov 0*8(%rsp), %rcx mov 1*8(%rsp), %r11 mov 5*8(%rsp), %rsp .endm /* Normal 64-bit system call target */ ENTRY(xen_syscall_target) undo_xen_syscall jmp system_call_after_swapgs ENDPROC(xen_syscall_target) #ifdef CONFIG_IA32_EMULATION /* 32-bit compat syscall target */ ENTRY(xen_syscall32_target) undo_xen_syscall jmp ia32_cstar_target ENDPROC(xen_syscall32_target) /* 32-bit compat sysenter target */ ENTRY(xen_sysenter_target) undo_xen_syscall jmp ia32_sysenter_target ENDPROC(xen_sysenter_target) #else /* !CONFIG_IA32_EMULATION */ ENTRY(xen_syscall32_target) ENTRY(xen_sysenter_target) lea 16(%rsp), %rsp /* strip %rcx, %r11 */ mov $-ENOSYS, %rax pushq $VGCF_in_syscall jmp hypercall_iret ENDPROC(xen_syscall32_target) ENDPROC(xen_sysenter_target) #endif /* CONFIG_IA32_EMULATION */