You've already forked linux-apfs
mirror of
https://github.com/linux-apfs/linux-apfs.git
synced 2026-05-01 15:00:59 -07:00
Merge branch 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 asm changes from Ingo Molnar:
"The main change in this cycle is another step in the big x86 system
call interface rework by Andy Lutomirski, which moves most of the low
level x86 entry code from assembly to C, for all syscall entries
except native 64-bit system calls:
arch/x86/entry/entry_32.S | 182 ++++------
arch/x86/entry/entry_64_compat.S | 547 ++++++++-----------------------
194 insertions(+), 535 deletions(-)
... our hope is that the final remaining step (converting native
64-bit system calls) will be less painful as all the previous steps,
given that most of the legacies and quirks are concentrated around
native 32-bit and compat environments"
* 'x86-asm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (47 commits)
x86/entry/32: Fix FS and GS restore in opportunistic SYSEXIT
x86/entry/32: Fix entry_INT80_32() to expect interrupts to be on
um/x86: Fix build after x86 syscall changes
x86/asm: Remove the xyz_cfi macros from dwarf2.h
selftests/x86: Style fixes for the 'unwind_vdso' test
x86/entry/64/compat: Document sysenter_fix_flags's reason for existence
x86/entry: Split and inline syscall_return_slowpath()
x86/entry: Split and inline prepare_exit_to_usermode()
x86/entry: Use pt_regs_to_thread_info() in syscall entry tracing
x86/entry: Hide two syscall entry assertions behind CONFIG_DEBUG_ENTRY
x86/entry: Micro-optimize compat fast syscall arg fetch
x86/entry: Force inlining of 32-bit syscall code
x86/entry: Make irqs_disabled checks in exit code depend on lockdep
x86/entry: Remove unnecessary IRQ twiddling in fast 32-bit syscalls
x86/asm: Remove thread_info.sysenter_return
x86/entry/32: Re-implement SYSENTER using the new C path
x86/entry/32: Switch INT80 to the new C syscall path
x86/entry/32: Open-code return tracking from fork and kthreads
x86/entry/compat: Implement opportunistic SYSRETL for compat syscalls
x86/vdso/compat: Wire up SYSENTER and SYSCSALL for compat userspace
...
This commit is contained in:
Linus Torvalds
@@ -2027,6 +2027,55 @@ config COMPAT_VDSO
|
||||
If unsure, say N: if you are compiling your own kernel, you
|
||||
are unlikely to be using a buggy version of glibc.
|
||||
|
||||
choice
|
||||
prompt "vsyscall table for legacy applications"
|
||||
depends on X86_64
|
||||
default LEGACY_VSYSCALL_EMULATE
|
||||
help
|
||||
Legacy user code that does not know how to find the vDSO expects
|
||||
to be able to issue three syscalls by calling fixed addresses in
|
||||
kernel space. Since this location is not randomized with ASLR,
|
||||
it can be used to assist security vulnerability exploitation.
|
||||
|
||||
This setting can be changed at boot time via the kernel command
|
||||
line parameter vsyscall=[native|emulate|none].
|
||||
|
||||
On a system with recent enough glibc (2.14 or newer) and no
|
||||
static binaries, you can say None without a performance penalty
|
||||
to improve security.
|
||||
|
||||
If unsure, select "Emulate".
|
||||
|
||||
config LEGACY_VSYSCALL_NATIVE
|
||||
bool "Native"
|
||||
help
|
||||
Actual executable code is located in the fixed vsyscall
|
||||
address mapping, implementing time() efficiently. Since
|
||||
this makes the mapping executable, it can be used during
|
||||
security vulnerability exploitation (traditionally as
|
||||
ROP gadgets). This configuration is not recommended.
|
||||
|
||||
config LEGACY_VSYSCALL_EMULATE
|
||||
bool "Emulate"
|
||||
help
|
||||
The kernel traps and emulates calls into the fixed
|
||||
vsyscall address mapping. This makes the mapping
|
||||
non-executable, but it still contains known contents,
|
||||
which could be used in certain rare security vulnerability
|
||||
exploits. This configuration is recommended when userspace
|
||||
still uses the vsyscall area.
|
||||
|
||||
config LEGACY_VSYSCALL_NONE
|
||||
bool "None"
|
||||
help
|
||||
There will be no vsyscall mapping at all. This will
|
||||
eliminate any risk of ASLR bypass due to the vsyscall
|
||||
fixed address mapping. Attempts to use the vsyscalls
|
||||
will be reported to dmesg, so that either old or
|
||||
malicious userspace programs can be identified.
|
||||
|
||||
endchoice
|
||||
|
||||
config CMDLINE_BOOL
|
||||
bool "Built-in kernel command line"
|
||||
---help---
|
||||
|
||||
+8
-2
@@ -159,6 +159,12 @@ endif
|
||||
sp-$(CONFIG_X86_32) := esp
|
||||
sp-$(CONFIG_X86_64) := rsp
|
||||
|
||||
# do binutils support CFI?
|
||||
cfi := $(call as-instr,.cfi_startproc\n.cfi_rel_offset $(sp-y)$(comma)0\n.cfi_endproc,-DCONFIG_AS_CFI=1)
|
||||
# is .cfi_signal_frame supported too?
|
||||
cfi-sigframe := $(call as-instr,.cfi_startproc\n.cfi_signal_frame\n.cfi_endproc,-DCONFIG_AS_CFI_SIGNAL_FRAME=1)
|
||||
cfi-sections := $(call as-instr,.cfi_sections .debug_frame,-DCONFIG_AS_CFI_SECTIONS=1)
|
||||
|
||||
# does binutils support specific instructions?
|
||||
asinstr := $(call as-instr,fxsaveq (%rax),-DCONFIG_AS_FXSAVEQ=1)
|
||||
asinstr += $(call as-instr,pshufb %xmm0$(comma)%xmm0,-DCONFIG_AS_SSSE3=1)
|
||||
@@ -166,8 +172,8 @@ asinstr += $(call as-instr,crc32l %eax$(comma)%eax,-DCONFIG_AS_CRC32=1)
|
||||
avx_instr := $(call as-instr,vxorps %ymm0$(comma)%ymm1$(comma)%ymm2,-DCONFIG_AS_AVX=1)
|
||||
avx2_instr :=$(call as-instr,vpbroadcastb %xmm0$(comma)%ymm1,-DCONFIG_AS_AVX2=1)
|
||||
|
||||
KBUILD_AFLAGS += $(asinstr) $(avx_instr) $(avx2_instr)
|
||||
KBUILD_CFLAGS += $(asinstr) $(avx_instr) $(avx2_instr)
|
||||
KBUILD_AFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr)
|
||||
KBUILD_CFLAGS += $(cfi) $(cfi-sigframe) $(cfi-sections) $(asinstr) $(avx_instr) $(avx2_instr)
|
||||
|
||||
LDFLAGS := -m elf_$(UTS_MACHINE)
|
||||
|
||||
|
||||
+216
-48
@@ -24,10 +24,19 @@
|
||||
|
||||
#include <asm/desc.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/vdso.h>
|
||||
#include <asm/uaccess.h>
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include <trace/events/syscalls.h>
|
||||
|
||||
static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long top_of_stack =
|
||||
(unsigned long)(regs + 1) + TOP_OF_KERNEL_STACK_PADDING;
|
||||
return (struct thread_info *)(top_of_stack - THREAD_SIZE);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CONTEXT_TRACKING
|
||||
/* Called on entry from user mode with IRQs off. */
|
||||
__visible void enter_from_user_mode(void)
|
||||
@@ -66,13 +75,14 @@ static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch)
|
||||
*/
|
||||
unsigned long syscall_trace_enter_phase1(struct pt_regs *regs, u32 arch)
|
||||
{
|
||||
struct thread_info *ti = pt_regs_to_thread_info(regs);
|
||||
unsigned long ret = 0;
|
||||
u32 work;
|
||||
|
||||
BUG_ON(regs != task_pt_regs(current));
|
||||
if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
|
||||
BUG_ON(regs != task_pt_regs(current));
|
||||
|
||||
work = ACCESS_ONCE(current_thread_info()->flags) &
|
||||
_TIF_WORK_SYSCALL_ENTRY;
|
||||
work = ACCESS_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;
|
||||
|
||||
#ifdef CONFIG_CONTEXT_TRACKING
|
||||
/*
|
||||
@@ -154,11 +164,12 @@ unsigned long syscall_trace_enter_phase1(struct pt_regs *regs, u32 arch)
|
||||
long syscall_trace_enter_phase2(struct pt_regs *regs, u32 arch,
|
||||
unsigned long phase1_result)
|
||||
{
|
||||
struct thread_info *ti = pt_regs_to_thread_info(regs);
|
||||
long ret = 0;
|
||||
u32 work = ACCESS_ONCE(current_thread_info()->flags) &
|
||||
_TIF_WORK_SYSCALL_ENTRY;
|
||||
u32 work = ACCESS_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY;
|
||||
|
||||
BUG_ON(regs != task_pt_regs(current));
|
||||
if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
|
||||
BUG_ON(regs != task_pt_regs(current));
|
||||
|
||||
/*
|
||||
* If we stepped into a sysenter/syscall insn, it trapped in
|
||||
@@ -207,19 +218,12 @@ long syscall_trace_enter(struct pt_regs *regs)
|
||||
return syscall_trace_enter_phase2(regs, arch, phase1_result);
|
||||
}
|
||||
|
||||
static struct thread_info *pt_regs_to_thread_info(struct pt_regs *regs)
|
||||
{
|
||||
unsigned long top_of_stack =
|
||||
(unsigned long)(regs + 1) + TOP_OF_KERNEL_STACK_PADDING;
|
||||
return (struct thread_info *)(top_of_stack - THREAD_SIZE);
|
||||
}
|
||||
#define EXIT_TO_USERMODE_LOOP_FLAGS \
|
||||
(_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_UPROBE | \
|
||||
_TIF_NEED_RESCHED | _TIF_USER_RETURN_NOTIFY)
|
||||
|
||||
/* Called with IRQs disabled. */
|
||||
__visible void prepare_exit_to_usermode(struct pt_regs *regs)
|
||||
static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags)
|
||||
{
|
||||
if (WARN_ON(!irqs_disabled()))
|
||||
local_irq_disable();
|
||||
|
||||
/*
|
||||
* In order to return to user mode, we need to have IRQs off with
|
||||
* none of _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_USER_RETURN_NOTIFY,
|
||||
@@ -229,14 +233,6 @@ __visible void prepare_exit_to_usermode(struct pt_regs *regs)
|
||||
* work to clear some of the flags can sleep.
|
||||
*/
|
||||
while (true) {
|
||||
u32 cached_flags =
|
||||
READ_ONCE(pt_regs_to_thread_info(regs)->flags);
|
||||
|
||||
if (!(cached_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME |
|
||||
_TIF_UPROBE | _TIF_NEED_RESCHED |
|
||||
_TIF_USER_RETURN_NOTIFY)))
|
||||
break;
|
||||
|
||||
/* We have work to do. */
|
||||
local_irq_enable();
|
||||
|
||||
@@ -260,50 +256,81 @@ __visible void prepare_exit_to_usermode(struct pt_regs *regs)
|
||||
|
||||
/* Disable IRQs and retry */
|
||||
local_irq_disable();
|
||||
|
||||
cached_flags = READ_ONCE(pt_regs_to_thread_info(regs)->flags);
|
||||
|
||||
if (!(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/* Called with IRQs disabled. */
|
||||
__visible inline void prepare_exit_to_usermode(struct pt_regs *regs)
|
||||
{
|
||||
u32 cached_flags;
|
||||
|
||||
if (IS_ENABLED(CONFIG_PROVE_LOCKING) && WARN_ON(!irqs_disabled()))
|
||||
local_irq_disable();
|
||||
|
||||
lockdep_sys_exit();
|
||||
|
||||
cached_flags =
|
||||
READ_ONCE(pt_regs_to_thread_info(regs)->flags);
|
||||
|
||||
if (unlikely(cached_flags & EXIT_TO_USERMODE_LOOP_FLAGS))
|
||||
exit_to_usermode_loop(regs, cached_flags);
|
||||
|
||||
user_enter();
|
||||
}
|
||||
|
||||
#define SYSCALL_EXIT_WORK_FLAGS \
|
||||
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
|
||||
_TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)
|
||||
|
||||
static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags)
|
||||
{
|
||||
bool step;
|
||||
|
||||
audit_syscall_exit(regs);
|
||||
|
||||
if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
|
||||
trace_sys_exit(regs, regs->ax);
|
||||
|
||||
/*
|
||||
* If TIF_SYSCALL_EMU is set, we only get here because of
|
||||
* TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
|
||||
* We already reported this syscall instruction in
|
||||
* syscall_trace_enter().
|
||||
*/
|
||||
step = unlikely(
|
||||
(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
|
||||
== _TIF_SINGLESTEP);
|
||||
if (step || cached_flags & _TIF_SYSCALL_TRACE)
|
||||
tracehook_report_syscall_exit(regs, step);
|
||||
}
|
||||
|
||||
/*
|
||||
* Called with IRQs on and fully valid regs. Returns with IRQs off in a
|
||||
* state such that we can immediately switch to user mode.
|
||||
*/
|
||||
__visible void syscall_return_slowpath(struct pt_regs *regs)
|
||||
__visible inline void syscall_return_slowpath(struct pt_regs *regs)
|
||||
{
|
||||
struct thread_info *ti = pt_regs_to_thread_info(regs);
|
||||
u32 cached_flags = READ_ONCE(ti->flags);
|
||||
bool step;
|
||||
|
||||
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
|
||||
|
||||
if (WARN(irqs_disabled(), "syscall %ld left IRQs disabled",
|
||||
regs->orig_ax))
|
||||
if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
|
||||
WARN(irqs_disabled(), "syscall %ld left IRQs disabled", regs->orig_ax))
|
||||
local_irq_enable();
|
||||
|
||||
/*
|
||||
* First do one-time work. If these work items are enabled, we
|
||||
* want to run them exactly once per syscall exit with IRQs on.
|
||||
*/
|
||||
if (cached_flags & (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT |
|
||||
_TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT)) {
|
||||
audit_syscall_exit(regs);
|
||||
|
||||
if (cached_flags & _TIF_SYSCALL_TRACEPOINT)
|
||||
trace_sys_exit(regs, regs->ax);
|
||||
|
||||
/*
|
||||
* If TIF_SYSCALL_EMU is set, we only get here because of
|
||||
* TIF_SINGLESTEP (i.e. this is PTRACE_SYSEMU_SINGLESTEP).
|
||||
* We already reported this syscall instruction in
|
||||
* syscall_trace_enter().
|
||||
*/
|
||||
step = unlikely(
|
||||
(cached_flags & (_TIF_SINGLESTEP | _TIF_SYSCALL_EMU))
|
||||
== _TIF_SINGLESTEP);
|
||||
if (step || cached_flags & _TIF_SYSCALL_TRACE)
|
||||
tracehook_report_syscall_exit(regs, step);
|
||||
}
|
||||
if (unlikely(cached_flags & SYSCALL_EXIT_WORK_FLAGS))
|
||||
syscall_slow_exit_work(regs, cached_flags);
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
/*
|
||||
@@ -316,3 +343,144 @@ __visible void syscall_return_slowpath(struct pt_regs *regs)
|
||||
local_irq_disable();
|
||||
prepare_exit_to_usermode(regs);
|
||||
}
|
||||
|
||||
#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
|
||||
/*
|
||||
* Does a 32-bit syscall. Called with IRQs on and does all entry and
|
||||
* exit work and returns with IRQs off. This function is extremely hot
|
||||
* in workloads that use it, and it's usually called from
|
||||
* do_fast_syscall_32, so forcibly inline it to improve performance.
|
||||
*/
|
||||
#ifdef CONFIG_X86_32
|
||||
/* 32-bit kernels use a trap gate for INT80, and the asm code calls here. */
|
||||
__visible
|
||||
#else
|
||||
/* 64-bit kernels use do_syscall_32_irqs_off() instead. */
|
||||
static
|
||||
#endif
|
||||
__always_inline void do_syscall_32_irqs_on(struct pt_regs *regs)
|
||||
{
|
||||
struct thread_info *ti = pt_regs_to_thread_info(regs);
|
||||
unsigned int nr = (unsigned int)regs->orig_ax;
|
||||
|
||||
#ifdef CONFIG_IA32_EMULATION
|
||||
ti->status |= TS_COMPAT;
|
||||
#endif
|
||||
|
||||
if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) {
|
||||
/*
|
||||
* Subtlety here: if ptrace pokes something larger than
|
||||
* 2^32-1 into orig_ax, this truncates it. This may or
|
||||
* may not be necessary, but it matches the old asm
|
||||
* behavior.
|
||||
*/
|
||||
nr = syscall_trace_enter(regs);
|
||||
}
|
||||
|
||||
if (likely(nr < IA32_NR_syscalls)) {
|
||||
/*
|
||||
* It's possible that a 32-bit syscall implementation
|
||||
* takes a 64-bit parameter but nonetheless assumes that
|
||||
* the high bits are zero. Make sure we zero-extend all
|
||||
* of the args.
|
||||
*/
|
||||
regs->ax = ia32_sys_call_table[nr](
|
||||
(unsigned int)regs->bx, (unsigned int)regs->cx,
|
||||
(unsigned int)regs->dx, (unsigned int)regs->si,
|
||||
(unsigned int)regs->di, (unsigned int)regs->bp);
|
||||
}
|
||||
|
||||
syscall_return_slowpath(regs);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/* Handles INT80 on 64-bit kernels */
|
||||
__visible void do_syscall_32_irqs_off(struct pt_regs *regs)
|
||||
{
|
||||
local_irq_enable();
|
||||
do_syscall_32_irqs_on(regs);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */
|
||||
__visible long do_fast_syscall_32(struct pt_regs *regs)
|
||||
{
|
||||
/*
|
||||
* Called using the internal vDSO SYSENTER/SYSCALL32 calling
|
||||
* convention. Adjust regs so it looks like we entered using int80.
|
||||
*/
|
||||
|
||||
unsigned long landing_pad = (unsigned long)current->mm->context.vdso +
|
||||
vdso_image_32.sym_int80_landing_pad;
|
||||
|
||||
/*
|
||||
* SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward
|
||||
* so that 'regs->ip -= 2' lands back on an int $0x80 instruction.
|
||||
* Fix it up.
|
||||
*/
|
||||
regs->ip = landing_pad;
|
||||
|
||||
/*
|
||||
* Fetch ECX from where the vDSO stashed it.
|
||||
*
|
||||
* WARNING: We are in CONTEXT_USER and RCU isn't paying attention!
|
||||
*/
|
||||
local_irq_enable();
|
||||
if (
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* Micro-optimization: the pointer we're following is explicitly
|
||||
* 32 bits, so it can't be out of range.
|
||||
*/
|
||||
__get_user(*(u32 *)®s->cx,
|
||||
(u32 __user __force *)(unsigned long)(u32)regs->sp)
|
||||
#else
|
||||
get_user(*(u32 *)®s->cx,
|
||||
(u32 __user __force *)(unsigned long)(u32)regs->sp)
|
||||
#endif
|
||||
) {
|
||||
|
||||
/* User code screwed up. */
|
||||
local_irq_disable();
|
||||
regs->ax = -EFAULT;
|
||||
#ifdef CONFIG_CONTEXT_TRACKING
|
||||
enter_from_user_mode();
|
||||
#endif
|
||||
prepare_exit_to_usermode(regs);
|
||||
return 0; /* Keep it simple: use IRET. */
|
||||
}
|
||||
|
||||
/* Now this is just like a normal syscall. */
|
||||
do_syscall_32_irqs_on(regs);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* Opportunistic SYSRETL: if possible, try to return using SYSRETL.
|
||||
* SYSRETL is available on all 64-bit CPUs, so we don't need to
|
||||
* bother with SYSEXIT.
|
||||
*
|
||||
* Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
|
||||
* because the ECX fixup above will ensure that this is essentially
|
||||
* never the case.
|
||||
*/
|
||||
return regs->cs == __USER32_CS && regs->ss == __USER_DS &&
|
||||
regs->ip == landing_pad &&
|
||||
(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF)) == 0;
|
||||
#else
|
||||
/*
|
||||
* Opportunistic SYSEXIT: if possible, try to return using SYSEXIT.
|
||||
*
|
||||
* Unlike 64-bit opportunistic SYSRET, we can't check that CX == IP,
|
||||
* because the ECX fixup above will ensure that this is essentially
|
||||
* never the case.
|
||||
*
|
||||
* We don't allow syscalls at all from VM86 mode, but we still
|
||||
* need to check VM, because we might be returning from sys_vm86.
|
||||
*/
|
||||
return static_cpu_has(X86_FEATURE_SEP) &&
|
||||
regs->cs == __USER_CS && regs->ss == __USER_DS &&
|
||||
regs->ip == landing_pad &&
|
||||
(regs->flags & (X86_EFLAGS_RF | X86_EFLAGS_TF | X86_EFLAGS_VM)) == 0;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
+61
-123
@@ -3,7 +3,7 @@
|
||||
*
|
||||
* entry_32.S contains the system-call and low-level fault and trap handling routines.
|
||||
*
|
||||
* Stack layout in 'syscall_exit':
|
||||
* Stack layout while running C code:
|
||||
* ptrace needs to have all registers on the stack.
|
||||
* If the order here is changed, it needs to be
|
||||
* updated in fork.c:copy_process(), signal.c:do_signal(),
|
||||
@@ -153,13 +153,13 @@
|
||||
|
||||
#endif /* CONFIG_X86_32_LAZY_GS */
|
||||
|
||||
.macro SAVE_ALL
|
||||
.macro SAVE_ALL pt_regs_ax=%eax
|
||||
cld
|
||||
PUSH_GS
|
||||
pushl %fs
|
||||
pushl %es
|
||||
pushl %ds
|
||||
pushl %eax
|
||||
pushl \pt_regs_ax
|
||||
pushl %ebp
|
||||
pushl %edi
|
||||
pushl %esi
|
||||
@@ -211,7 +211,11 @@ ENTRY(ret_from_fork)
|
||||
popl %eax
|
||||
pushl $0x0202 # Reset kernel eflags
|
||||
popfl
|
||||
jmp syscall_exit
|
||||
|
||||
/* When we fork, we trace the syscall return in the child, too. */
|
||||
movl %esp, %eax
|
||||
call syscall_return_slowpath
|
||||
jmp restore_all
|
||||
END(ret_from_fork)
|
||||
|
||||
ENTRY(ret_from_kernel_thread)
|
||||
@@ -224,7 +228,15 @@ ENTRY(ret_from_kernel_thread)
|
||||
movl PT_EBP(%esp), %eax
|
||||
call *PT_EBX(%esp)
|
||||
movl $0, PT_EAX(%esp)
|
||||
jmp syscall_exit
|
||||
|
||||
/*
|
||||
* Kernel threads return to userspace as if returning from a syscall.
|
||||
* We should check whether anything actually uses this path and, if so,
|
||||
* consider switching it over to ret_from_fork.
|
||||
*/
|
||||
movl %esp, %eax
|
||||
call syscall_return_slowpath
|
||||
jmp restore_all
|
||||
ENDPROC(ret_from_kernel_thread)
|
||||
|
||||
/*
|
||||
@@ -255,7 +267,6 @@ ret_from_intr:
|
||||
jb resume_kernel # not returning to v8086 or userspace
|
||||
|
||||
ENTRY(resume_userspace)
|
||||
LOCKDEP_SYS_EXIT
|
||||
DISABLE_INTERRUPTS(CLBR_ANY)
|
||||
TRACE_IRQS_OFF
|
||||
movl %esp, %eax
|
||||
@@ -276,76 +287,47 @@ need_resched:
|
||||
END(resume_kernel)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* SYSENTER_RETURN points to after the SYSENTER instruction
|
||||
* in the vsyscall page. See vsyscall-sysentry.S, which defines
|
||||
* the symbol.
|
||||
*/
|
||||
|
||||
# SYSENTER call handler stub
|
||||
ENTRY(entry_SYSENTER_32)
|
||||
movl TSS_sysenter_sp0(%esp), %esp
|
||||
sysenter_past_esp:
|
||||
pushl $__USER_DS /* pt_regs->ss */
|
||||
pushl %ecx /* pt_regs->cx */
|
||||
pushfl /* pt_regs->flags (except IF = 0) */
|
||||
orl $X86_EFLAGS_IF, (%esp) /* Fix IF */
|
||||
pushl $__USER_CS /* pt_regs->cs */
|
||||
pushl $0 /* pt_regs->ip = 0 (placeholder) */
|
||||
pushl %eax /* pt_regs->orig_ax */
|
||||
SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
|
||||
|
||||
/*
|
||||
* Interrupts are disabled here, but we can't trace it until
|
||||
* enough kernel state to call TRACE_IRQS_OFF can be called - but
|
||||
* we immediately enable interrupts at that point anyway.
|
||||
* User mode is traced as though IRQs are on, and SYSENTER
|
||||
* turned them off.
|
||||
*/
|
||||
pushl $__USER_DS
|
||||
pushl %ebp
|
||||
pushfl
|
||||
orl $X86_EFLAGS_IF, (%esp)
|
||||
pushl $__USER_CS
|
||||
/*
|
||||
* Push current_thread_info()->sysenter_return to the stack.
|
||||
* A tiny bit of offset fixup is necessary: TI_sysenter_return
|
||||
* is relative to thread_info, which is at the bottom of the
|
||||
* kernel stack page. 4*4 means the 4 words pushed above;
|
||||
* TOP_OF_KERNEL_STACK_PADDING takes us to the top of the stack;
|
||||
* and THREAD_SIZE takes us to the bottom.
|
||||
*/
|
||||
pushl ((TI_sysenter_return) - THREAD_SIZE + TOP_OF_KERNEL_STACK_PADDING + 4*4)(%esp)
|
||||
|
||||
pushl %eax
|
||||
SAVE_ALL
|
||||
ENABLE_INTERRUPTS(CLBR_NONE)
|
||||
|
||||
/*
|
||||
* Load the potential sixth argument from user stack.
|
||||
* Careful about security.
|
||||
*/
|
||||
cmpl $__PAGE_OFFSET-3, %ebp
|
||||
jae syscall_fault
|
||||
ASM_STAC
|
||||
1: movl (%ebp), %ebp
|
||||
ASM_CLAC
|
||||
movl %ebp, PT_EBP(%esp)
|
||||
_ASM_EXTABLE(1b, syscall_fault)
|
||||
|
||||
GET_THREAD_INFO(%ebp)
|
||||
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp)
|
||||
jnz syscall_trace_entry
|
||||
sysenter_do_call:
|
||||
cmpl $(NR_syscalls), %eax
|
||||
jae sysenter_badsys
|
||||
call *sys_call_table(, %eax, 4)
|
||||
sysenter_after_call:
|
||||
movl %eax, PT_EAX(%esp)
|
||||
LOCKDEP_SYS_EXIT
|
||||
DISABLE_INTERRUPTS(CLBR_ANY)
|
||||
TRACE_IRQS_OFF
|
||||
movl TI_flags(%ebp), %ecx
|
||||
testl $_TIF_ALLWORK_MASK, %ecx
|
||||
jnz syscall_exit_work_irqs_off
|
||||
sysenter_exit:
|
||||
/* if something modifies registers it must also disable sysexit */
|
||||
movl PT_EIP(%esp), %edx
|
||||
movl PT_OLDESP(%esp), %ecx
|
||||
xorl %ebp, %ebp
|
||||
TRACE_IRQS_ON
|
||||
|
||||
movl %esp, %eax
|
||||
call do_fast_syscall_32
|
||||
testl %eax, %eax
|
||||
jz .Lsyscall_32_done
|
||||
|
||||
/* Opportunistic SYSEXIT */
|
||||
TRACE_IRQS_ON /* User mode traces as IRQs on. */
|
||||
movl PT_EIP(%esp), %edx /* pt_regs->ip */
|
||||
movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
|
||||
1: mov PT_FS(%esp), %fs
|
||||
PTGS_TO_GS
|
||||
popl %ebx /* pt_regs->bx */
|
||||
addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
|
||||
popl %esi /* pt_regs->si */
|
||||
popl %edi /* pt_regs->di */
|
||||
popl %ebp /* pt_regs->bp */
|
||||
popl %eax /* pt_regs->ax */
|
||||
|
||||
/*
|
||||
* Return back to the vDSO, which will pop ecx and edx.
|
||||
* Don't bother with DS and ES (they already contain __USER_DS).
|
||||
*/
|
||||
ENABLE_INTERRUPTS_SYSEXIT
|
||||
|
||||
.pushsection .fixup, "ax"
|
||||
@@ -359,21 +341,18 @@ ENDPROC(entry_SYSENTER_32)
|
||||
# system call handler stub
|
||||
ENTRY(entry_INT80_32)
|
||||
ASM_CLAC
|
||||
pushl %eax # save orig_eax
|
||||
SAVE_ALL
|
||||
GET_THREAD_INFO(%ebp)
|
||||
# system call tracing in operation / emulation
|
||||
testl $_TIF_WORK_SYSCALL_ENTRY, TI_flags(%ebp)
|
||||
jnz syscall_trace_entry
|
||||
cmpl $(NR_syscalls), %eax
|
||||
jae syscall_badsys
|
||||
syscall_call:
|
||||
call *sys_call_table(, %eax, 4)
|
||||
syscall_after_call:
|
||||
movl %eax, PT_EAX(%esp) # store the return value
|
||||
syscall_exit:
|
||||
LOCKDEP_SYS_EXIT
|
||||
jmp syscall_exit_work
|
||||
pushl %eax /* pt_regs->orig_ax */
|
||||
SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
|
||||
|
||||
/*
|
||||
* User mode is traced as though IRQs are on. Unlike the 64-bit
|
||||
* case, INT80 is a trap gate on 32-bit kernels, so interrupts
|
||||
* are already on (unless user code is messing around with iopl).
|
||||
*/
|
||||
|
||||
movl %esp, %eax
|
||||
call do_syscall_32_irqs_on
|
||||
.Lsyscall_32_done:
|
||||
|
||||
restore_all:
|
||||
TRACE_IRQS_IRET
|
||||
@@ -450,47 +429,6 @@ ldt_ss:
|
||||
#endif
|
||||
ENDPROC(entry_INT80_32)
|
||||
|
||||
# perform syscall exit tracing
|
||||
ALIGN
|
||||
syscall_trace_entry:
|
||||
movl $-ENOSYS, PT_EAX(%esp)
|
||||
movl %esp, %eax
|
||||
call syscall_trace_enter
|
||||
/* What it returned is what we'll actually use. */
|
||||
cmpl $(NR_syscalls), %eax
|
||||
jnae syscall_call
|
||||
jmp syscall_exit
|
||||
END(syscall_trace_entry)
|
||||
|
||||
# perform syscall exit tracing
|
||||
ALIGN
|
||||
syscall_exit_work_irqs_off:
|
||||
TRACE_IRQS_ON
|
||||
ENABLE_INTERRUPTS(CLBR_ANY)
|
||||
|
||||
syscall_exit_work:
|
||||
movl %esp, %eax
|
||||
call syscall_return_slowpath
|
||||
jmp restore_all
|
||||
END(syscall_exit_work)
|
||||
|
||||
syscall_fault:
|
||||
ASM_CLAC
|
||||
GET_THREAD_INFO(%ebp)
|
||||
movl $-EFAULT, PT_EAX(%esp)
|
||||
jmp resume_userspace
|
||||
END(syscall_fault)
|
||||
|
||||
syscall_badsys:
|
||||
movl $-ENOSYS, %eax
|
||||
jmp syscall_after_call
|
||||
END(syscall_badsys)
|
||||
|
||||
sysenter_badsys:
|
||||
movl $-ENOSYS, %eax
|
||||
jmp sysenter_after_call
|
||||
END(sysenter_badsys)
|
||||
|
||||
.macro FIXUP_ESPFIX_STACK
|
||||
/*
|
||||
* Switch back for ESPFIX stack to the normal zerobased stack
|
||||
|
||||
@@ -391,20 +391,16 @@ GLOBAL(stub_execveat)
|
||||
jmp return_from_execve
|
||||
END(stub_execveat)
|
||||
|
||||
#if defined(CONFIG_X86_X32_ABI) || defined(CONFIG_IA32_EMULATION)
|
||||
#if defined(CONFIG_X86_X32_ABI)
|
||||
.align 8
|
||||
GLOBAL(stub_x32_execve)
|
||||
GLOBAL(stub32_execve)
|
||||
call compat_sys_execve
|
||||
jmp return_from_execve
|
||||
END(stub32_execve)
|
||||
END(stub_x32_execve)
|
||||
.align 8
|
||||
GLOBAL(stub_x32_execveat)
|
||||
GLOBAL(stub32_execveat)
|
||||
call compat_sys_execveat
|
||||
jmp return_from_execve
|
||||
END(stub32_execveat)
|
||||
END(stub_x32_execveat)
|
||||
#endif
|
||||
|
||||
@@ -557,7 +553,6 @@ ret_from_intr:
|
||||
jz retint_kernel
|
||||
|
||||
/* Interrupt came from user space */
|
||||
LOCKDEP_SYS_EXIT_IRQ
|
||||
GLOBAL(retint_user)
|
||||
mov %rsp,%rdi
|
||||
call prepare_exit_to_usermode
|
||||
@@ -587,7 +582,7 @@ retint_kernel:
|
||||
* At this label, code paths which return to kernel and to user,
|
||||
* which come from interrupts/exception and from syscalls, merge.
|
||||
*/
|
||||
restore_regs_and_iret:
|
||||
GLOBAL(restore_regs_and_iret)
|
||||
RESTORE_EXTRA_REGS
|
||||
restore_c_regs_and_iret:
|
||||
RESTORE_C_REGS
|
||||
|
||||
+137
-416
File diff suppressed because it is too large
Load Diff
@@ -4,24 +4,21 @@
|
||||
#include <linux/sys.h>
|
||||
#include <linux/cache.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/syscall.h>
|
||||
|
||||
#ifdef CONFIG_IA32_EMULATION
|
||||
#define SYM(sym, compat) compat
|
||||
#else
|
||||
#define SYM(sym, compat) sym
|
||||
#define ia32_sys_call_table sys_call_table
|
||||
#define __NR_syscall_compat_max __NR_syscall_max
|
||||
#endif
|
||||
|
||||
#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage void SYM(sym, compat)(void) ;
|
||||
#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage long SYM(sym, compat)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
|
||||
#include <asm/syscalls_32.h>
|
||||
#undef __SYSCALL_I386
|
||||
|
||||
#define __SYSCALL_I386(nr, sym, compat) [nr] = SYM(sym, compat),
|
||||
|
||||
typedef asmlinkage void (*sys_call_ptr_t)(void);
|
||||
|
||||
extern asmlinkage void sys_ni_syscall(void);
|
||||
extern asmlinkage long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
|
||||
|
||||
__visible const sys_call_ptr_t ia32_sys_call_table[__NR_syscall_compat_max+1] = {
|
||||
/*
|
||||
|
||||
@@ -14,13 +14,13 @@
|
||||
# define __SYSCALL_X32(nr, sym, compat) /* nothing */
|
||||
#endif
|
||||
|
||||
#define __SYSCALL_64(nr, sym, compat) extern asmlinkage void sym(void) ;
|
||||
#define __SYSCALL_64(nr, sym, compat) extern asmlinkage long sym(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) ;
|
||||
#include <asm/syscalls_64.h>
|
||||
#undef __SYSCALL_64
|
||||
|
||||
#define __SYSCALL_64(nr, sym, compat) [nr] = sym,
|
||||
|
||||
extern void sys_ni_syscall(void);
|
||||
extern long sys_ni_syscall(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long);
|
||||
|
||||
asmlinkage const sys_call_ptr_t sys_call_table[__NR_syscall_max+1] = {
|
||||
/*
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
#
|
||||
0 i386 restart_syscall sys_restart_syscall
|
||||
1 i386 exit sys_exit
|
||||
2 i386 fork sys_fork stub32_fork
|
||||
2 i386 fork sys_fork sys_fork
|
||||
3 i386 read sys_read
|
||||
4 i386 write sys_write
|
||||
5 i386 open sys_open compat_sys_open
|
||||
@@ -17,7 +17,7 @@
|
||||
8 i386 creat sys_creat
|
||||
9 i386 link sys_link
|
||||
10 i386 unlink sys_unlink
|
||||
11 i386 execve sys_execve stub32_execve
|
||||
11 i386 execve sys_execve compat_sys_execve
|
||||
12 i386 chdir sys_chdir
|
||||
13 i386 time sys_time compat_sys_time
|
||||
14 i386 mknod sys_mknod
|
||||
@@ -125,7 +125,7 @@
|
||||
116 i386 sysinfo sys_sysinfo compat_sys_sysinfo
|
||||
117 i386 ipc sys_ipc compat_sys_ipc
|
||||
118 i386 fsync sys_fsync
|
||||
119 i386 sigreturn sys_sigreturn stub32_sigreturn
|
||||
119 i386 sigreturn sys_sigreturn sys32_sigreturn
|
||||
120 i386 clone sys_clone stub32_clone
|
||||
121 i386 setdomainname sys_setdomainname
|
||||
122 i386 uname sys_newuname
|
||||
@@ -179,7 +179,7 @@
|
||||
170 i386 setresgid sys_setresgid16
|
||||
171 i386 getresgid sys_getresgid16
|
||||
172 i386 prctl sys_prctl
|
||||
173 i386 rt_sigreturn sys_rt_sigreturn stub32_rt_sigreturn
|
||||
173 i386 rt_sigreturn sys_rt_sigreturn sys32_rt_sigreturn
|
||||
174 i386 rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction
|
||||
175 i386 rt_sigprocmask sys_rt_sigprocmask
|
||||
176 i386 rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending
|
||||
@@ -196,7 +196,7 @@
|
||||
187 i386 sendfile sys_sendfile compat_sys_sendfile
|
||||
188 i386 getpmsg
|
||||
189 i386 putpmsg
|
||||
190 i386 vfork sys_vfork stub32_vfork
|
||||
190 i386 vfork sys_vfork sys_vfork
|
||||
191 i386 ugetrlimit sys_getrlimit compat_sys_getrlimit
|
||||
192 i386 mmap2 sys_mmap_pgoff
|
||||
193 i386 truncate64 sys_truncate64 sys32_truncate64
|
||||
@@ -364,7 +364,7 @@
|
||||
355 i386 getrandom sys_getrandom
|
||||
356 i386 memfd_create sys_memfd_create
|
||||
357 i386 bpf sys_bpf
|
||||
358 i386 execveat sys_execveat stub32_execveat
|
||||
358 i386 execveat sys_execveat compat_sys_execveat
|
||||
359 i386 socket sys_socket
|
||||
360 i386 socketpair sys_socketpair
|
||||
361 i386 bind sys_bind
|
||||
|
||||
@@ -19,9 +19,7 @@ obj-y += vma.o
|
||||
# vDSO images to build
|
||||
vdso_img-$(VDSO64-y) += 64
|
||||
vdso_img-$(VDSOX32-y) += x32
|
||||
vdso_img-$(VDSO32-y) += 32-int80
|
||||
vdso_img-$(CONFIG_IA32_EMULATION) += 32-syscall
|
||||
vdso_img-$(VDSO32-y) += 32-sysenter
|
||||
vdso_img-$(VDSO32-y) += 32
|
||||
|
||||
obj-$(VDSO32-y) += vdso32-setup.o
|
||||
|
||||
@@ -69,7 +67,7 @@ $(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE
|
||||
CFL := $(PROFILING) -mcmodel=small -fPIC -O2 -fasynchronous-unwind-tables -m64 \
|
||||
$(filter -g%,$(KBUILD_CFLAGS)) $(call cc-option, -fno-stack-protector) \
|
||||
-fno-omit-frame-pointer -foptimize-sibling-calls \
|
||||
-DDISABLE_BRANCH_PROFILING
|
||||
-DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
|
||||
|
||||
$(vobjs): KBUILD_CFLAGS += $(CFL)
|
||||
|
||||
@@ -122,15 +120,6 @@ $(obj)/%.so: $(obj)/%.so.dbg
|
||||
$(obj)/vdsox32.so.dbg: $(src)/vdsox32.lds $(vobjx32s) FORCE
|
||||
$(call if_changed,vdso)
|
||||
|
||||
#
|
||||
# Build multiple 32-bit vDSO images to choose from at boot time.
|
||||
#
|
||||
vdso32.so-$(VDSO32-y) += int80
|
||||
vdso32.so-$(CONFIG_IA32_EMULATION) += syscall
|
||||
vdso32.so-$(VDSO32-y) += sysenter
|
||||
|
||||
vdso32-images = $(vdso32.so-y:%=vdso32-%.so)
|
||||
|
||||
CPPFLAGS_vdso32.lds = $(CPPFLAGS_vdso.lds)
|
||||
VDSO_LDFLAGS_vdso32.lds = -m32 -Wl,-m,elf_i386 -Wl,-soname=linux-gate.so.1
|
||||
|
||||
@@ -139,14 +128,12 @@ VDSO_LDFLAGS_vdso32.lds = -m32 -Wl,-m,elf_i386 -Wl,-soname=linux-gate.so.1
|
||||
override obj-dirs = $(dir $(obj)) $(obj)/vdso32/
|
||||
|
||||
targets += vdso32/vdso32.lds
|
||||
targets += vdso32/note.o vdso32/vclock_gettime.o $(vdso32.so-y:%=vdso32/%.o)
|
||||
targets += vdso32/note.o vdso32/vclock_gettime.o vdso32/system_call.o
|
||||
targets += vdso32/vclock_gettime.o
|
||||
|
||||
$(obj)/vdso32.o: $(vdso32-images:%=$(obj)/%)
|
||||
|
||||
KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS))
|
||||
$(vdso32-images:%=$(obj)/%.dbg): KBUILD_AFLAGS = $(KBUILD_AFLAGS_32)
|
||||
$(vdso32-images:%=$(obj)/%.dbg): asflags-$(CONFIG_X86_64) += -m32
|
||||
KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) -DBUILD_VDSO
|
||||
$(obj)/vdso32.so.dbg: KBUILD_AFLAGS = $(KBUILD_AFLAGS_32)
|
||||
$(obj)/vdso32.so.dbg: asflags-$(CONFIG_X86_64) += -m32
|
||||
|
||||
KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS))
|
||||
KBUILD_CFLAGS_32 := $(filter-out -mcmodel=kernel,$(KBUILD_CFLAGS_32))
|
||||
@@ -157,13 +144,13 @@ KBUILD_CFLAGS_32 += $(call cc-option, -fno-stack-protector)
|
||||
KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)
|
||||
KBUILD_CFLAGS_32 += -fno-omit-frame-pointer
|
||||
KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING
|
||||
$(vdso32-images:%=$(obj)/%.dbg): KBUILD_CFLAGS = $(KBUILD_CFLAGS_32)
|
||||
$(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32)
|
||||
|
||||
$(vdso32-images:%=$(obj)/%.dbg): $(obj)/vdso32-%.so.dbg: FORCE \
|
||||
$(obj)/vdso32/vdso32.lds \
|
||||
$(obj)/vdso32/vclock_gettime.o \
|
||||
$(obj)/vdso32/note.o \
|
||||
$(obj)/vdso32/%.o
|
||||
$(obj)/vdso32.so.dbg: FORCE \
|
||||
$(obj)/vdso32/vdso32.lds \
|
||||
$(obj)/vdso32/vclock_gettime.o \
|
||||
$(obj)/vdso32/note.o \
|
||||
$(obj)/vdso32/system_call.o
|
||||
$(call if_changed,vdso)
|
||||
|
||||
#
|
||||
@@ -206,4 +193,4 @@ $(vdso_img_insttargets): install_%: $(obj)/%.dbg $(MODLIB)/vdso FORCE
|
||||
PHONY += vdso_install $(vdso_img_insttargets)
|
||||
vdso_install: $(vdso_img_insttargets) FORCE
|
||||
|
||||
clean-files := vdso32-syscall* vdso32-sysenter* vdso32-int80* vdso64* vdso-image-*.c vdsox32.so*
|
||||
clean-files := vdso32.so vdso32.so.dbg vdso64* vdso-image-*.c vdsox32.so*
|
||||
|
||||
@@ -98,10 +98,10 @@ struct vdso_sym required_syms[] = {
|
||||
"VDSO_FAKE_SECTION_TABLE_END", false
|
||||
},
|
||||
{"VDSO32_NOTE_MASK", true},
|
||||
{"VDSO32_SYSENTER_RETURN", true},
|
||||
{"__kernel_vsyscall", true},
|
||||
{"__kernel_sigreturn", true},
|
||||
{"__kernel_rt_sigreturn", true},
|
||||
{"int80_landing_pad", true},
|
||||
};
|
||||
|
||||
__attribute__((format(printf, 1, 2))) __attribute__((noreturn))
|
||||
|
||||
@@ -48,35 +48,9 @@ __setup("vdso32=", vdso32_setup);
|
||||
__setup_param("vdso=", vdso_setup, vdso32_setup, 0);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
||||
#define vdso32_sysenter() (boot_cpu_has(X86_FEATURE_SYSENTER32))
|
||||
#define vdso32_syscall() (boot_cpu_has(X86_FEATURE_SYSCALL32))
|
||||
|
||||
#else /* CONFIG_X86_32 */
|
||||
|
||||
#define vdso32_sysenter() (boot_cpu_has(X86_FEATURE_SEP))
|
||||
#define vdso32_syscall() (0)
|
||||
|
||||
#endif /* CONFIG_X86_64 */
|
||||
|
||||
#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
|
||||
const struct vdso_image *selected_vdso32;
|
||||
#endif
|
||||
|
||||
int __init sysenter_setup(void)
|
||||
{
|
||||
#ifdef CONFIG_COMPAT
|
||||
if (vdso32_syscall())
|
||||
selected_vdso32 = &vdso_image_32_syscall;
|
||||
else
|
||||
#endif
|
||||
if (vdso32_sysenter())
|
||||
selected_vdso32 = &vdso_image_32_sysenter;
|
||||
else
|
||||
selected_vdso32 = &vdso_image_32_int80;
|
||||
|
||||
init_vdso_image(selected_vdso32);
|
||||
init_vdso_image(&vdso_image_32);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
/*
|
||||
* Code for the vDSO. This version uses the old int $0x80 method.
|
||||
*
|
||||
* First get the common code for the sigreturn entry points.
|
||||
* This must come first.
|
||||
*/
|
||||
#include "sigreturn.S"
|
||||
|
||||
.text
|
||||
.globl __kernel_vsyscall
|
||||
.type __kernel_vsyscall,@function
|
||||
ALIGN
|
||||
__kernel_vsyscall:
|
||||
.LSTART_vsyscall:
|
||||
int $0x80
|
||||
ret
|
||||
.LEND_vsyscall:
|
||||
.size __kernel_vsyscall,.-.LSTART_vsyscall
|
||||
.previous
|
||||
|
||||
.section .eh_frame,"a",@progbits
|
||||
.LSTARTFRAMEDLSI:
|
||||
.long .LENDCIEDLSI-.LSTARTCIEDLSI
|
||||
.LSTARTCIEDLSI:
|
||||
.long 0 /* CIE ID */
|
||||
.byte 1 /* Version number */
|
||||
.string "zR" /* NUL-terminated augmentation string */
|
||||
.uleb128 1 /* Code alignment factor */
|
||||
.sleb128 -4 /* Data alignment factor */
|
||||
.byte 8 /* Return address register column */
|
||||
.uleb128 1 /* Augmentation value length */
|
||||
.byte 0x1b /* DW_EH_PE_pcrel|DW_EH_PE_sdata4. */
|
||||
.byte 0x0c /* DW_CFA_def_cfa */
|
||||
.uleb128 4
|
||||
.uleb128 4
|
||||
.byte 0x88 /* DW_CFA_offset, column 0x8 */
|
||||
.uleb128 1
|
||||
.align 4
|
||||
.LENDCIEDLSI:
|
||||
.long .LENDFDEDLSI-.LSTARTFDEDLSI /* Length FDE */
|
||||
.LSTARTFDEDLSI:
|
||||
.long .LSTARTFDEDLSI-.LSTARTFRAMEDLSI /* CIE pointer */
|
||||
.long .LSTART_vsyscall-. /* PC-relative start address */
|
||||
.long .LEND_vsyscall-.LSTART_vsyscall
|
||||
.uleb128 0
|
||||
.align 4
|
||||
.LENDFDEDLSI:
|
||||
.previous
|
||||
|
||||
/*
|
||||
* Pad out the segment to match the size of the sysenter.S version.
|
||||
*/
|
||||
VDSO32_vsyscall_eh_frame_size = 0x40
|
||||
.section .data,"aw",@progbits
|
||||
.space VDSO32_vsyscall_eh_frame_size-(.LENDFDEDLSI-.LSTARTFRAMEDLSI), 0
|
||||
.previous
|
||||
@@ -1,75 +0,0 @@
|
||||
/*
|
||||
* Code for the vDSO. This version uses the syscall instruction.
|
||||
*
|
||||
* First get the common code for the sigreturn entry points.
|
||||
* This must come first.
|
||||
*/
|
||||
#define SYSCALL_ENTER_KERNEL syscall
|
||||
#include "sigreturn.S"
|
||||
|
||||
#include <asm/segment.h>
|
||||
|
||||
.text
|
||||
.globl __kernel_vsyscall
|
||||
.type __kernel_vsyscall,@function
|
||||
ALIGN
|
||||
__kernel_vsyscall:
|
||||
.LSTART_vsyscall:
|
||||
push %ebp
|
||||
.Lpush_ebp:
|
||||
movl %ecx, %ebp
|
||||
syscall
|
||||
movl %ebp, %ecx
|
||||
popl %ebp
|
||||
.Lpop_ebp:
|
||||
ret
|
||||
.LEND_vsyscall:
|
||||
.size __kernel_vsyscall,.-.LSTART_vsyscall
|
||||
|
||||
.section .eh_frame,"a",@progbits
|
||||
.LSTARTFRAME:
|
||||
.long .LENDCIE-.LSTARTCIE
|
||||
.LSTARTCIE:
|
||||
.long 0 /* CIE ID */
|
||||
.byte 1 /* Version number */
|
||||
.string "zR" /* NUL-terminated augmentation string */
|
||||
.uleb128 1 /* Code alignment factor */
|
||||
.sleb128 -4 /* Data alignment factor */
|
||||
.byte 8 /* Return address register column */
|
||||
.uleb128 1 /* Augmentation value length */
|
||||
.byte 0x1b /* DW_EH_PE_pcrel|DW_EH_PE_sdata4. */
|
||||
.byte 0x0c /* DW_CFA_def_cfa */
|
||||
.uleb128 4
|
||||
.uleb128 4
|
||||
.byte 0x88 /* DW_CFA_offset, column 0x8 */
|
||||
.uleb128 1
|
||||
.align 4
|
||||
.LENDCIE:
|
||||
|
||||
.long .LENDFDE1-.LSTARTFDE1 /* Length FDE */
|
||||
.LSTARTFDE1:
|
||||
.long .LSTARTFDE1-.LSTARTFRAME /* CIE pointer */
|
||||
.long .LSTART_vsyscall-. /* PC-relative start address */
|
||||
.long .LEND_vsyscall-.LSTART_vsyscall
|
||||
.uleb128 0 /* Augmentation length */
|
||||
/* What follows are the instructions for the table generation.
|
||||
We have to record all changes of the stack pointer. */
|
||||
.byte 0x40 + .Lpush_ebp-.LSTART_vsyscall /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.uleb128 8
|
||||
.byte 0x85, 0x02 /* DW_CFA_offset %ebp -8 */
|
||||
.byte 0x40 + .Lpop_ebp-.Lpush_ebp /* DW_CFA_advance_loc */
|
||||
.byte 0xc5 /* DW_CFA_restore %ebp */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.uleb128 4
|
||||
.align 4
|
||||
.LENDFDE1:
|
||||
.previous
|
||||
|
||||
/*
|
||||
* Pad out the segment to match the size of the sysenter.S version.
|
||||
*/
|
||||
VDSO32_vsyscall_eh_frame_size = 0x40
|
||||
.section .data,"aw",@progbits
|
||||
.space VDSO32_vsyscall_eh_frame_size-(.LENDFDE1-.LSTARTFRAME), 0
|
||||
.previous
|
||||
@@ -1,116 +0,0 @@
|
||||
/*
|
||||
* Code for the vDSO. This version uses the sysenter instruction.
|
||||
*
|
||||
* First get the common code for the sigreturn entry points.
|
||||
* This must come first.
|
||||
*/
|
||||
#include "sigreturn.S"
|
||||
|
||||
/*
|
||||
* The caller puts arg2 in %ecx, which gets pushed. The kernel will use
|
||||
* %ecx itself for arg2. The pushing is because the sysexit instruction
|
||||
* (found in entry.S) requires that we clobber %ecx with the desired %esp.
|
||||
* User code might expect that %ecx is unclobbered though, as it would be
|
||||
* for returning via the iret instruction, so we must push and pop.
|
||||
*
|
||||
* The caller puts arg3 in %edx, which the sysexit instruction requires
|
||||
* for %eip. Thus, exactly as for arg2, we must push and pop.
|
||||
*
|
||||
* Arg6 is different. The caller puts arg6 in %ebp. Since the sysenter
|
||||
* instruction clobbers %esp, the user's %esp won't even survive entry
|
||||
* into the kernel. We store %esp in %ebp. Code in entry.S must fetch
|
||||
* arg6 from the stack.
|
||||
*
|
||||
* You can not use this vsyscall for the clone() syscall because the
|
||||
* three words on the parent stack do not get copied to the child.
|
||||
*/
|
||||
.text
|
||||
.globl __kernel_vsyscall
|
||||
.type __kernel_vsyscall,@function
|
||||
ALIGN
|
||||
__kernel_vsyscall:
|
||||
.LSTART_vsyscall:
|
||||
push %ecx
|
||||
.Lpush_ecx:
|
||||
push %edx
|
||||
.Lpush_edx:
|
||||
push %ebp
|
||||
.Lenter_kernel:
|
||||
movl %esp,%ebp
|
||||
sysenter
|
||||
|
||||
/* 7: align return point with nop's to make disassembly easier */
|
||||
.space 7,0x90
|
||||
|
||||
/* 14: System call restart point is here! (SYSENTER_RETURN-2) */
|
||||
int $0x80
|
||||
/* 16: System call normal return point is here! */
|
||||
VDSO32_SYSENTER_RETURN: /* Symbol used by sysenter.c via vdso32-syms.h */
|
||||
pop %ebp
|
||||
.Lpop_ebp:
|
||||
pop %edx
|
||||
.Lpop_edx:
|
||||
pop %ecx
|
||||
.Lpop_ecx:
|
||||
ret
|
||||
.LEND_vsyscall:
|
||||
.size __kernel_vsyscall,.-.LSTART_vsyscall
|
||||
.previous
|
||||
|
||||
.section .eh_frame,"a",@progbits
|
||||
.LSTARTFRAMEDLSI:
|
||||
.long .LENDCIEDLSI-.LSTARTCIEDLSI
|
||||
.LSTARTCIEDLSI:
|
||||
.long 0 /* CIE ID */
|
||||
.byte 1 /* Version number */
|
||||
.string "zR" /* NUL-terminated augmentation string */
|
||||
.uleb128 1 /* Code alignment factor */
|
||||
.sleb128 -4 /* Data alignment factor */
|
||||
.byte 8 /* Return address register column */
|
||||
.uleb128 1 /* Augmentation value length */
|
||||
.byte 0x1b /* DW_EH_PE_pcrel|DW_EH_PE_sdata4. */
|
||||
.byte 0x0c /* DW_CFA_def_cfa */
|
||||
.uleb128 4
|
||||
.uleb128 4
|
||||
.byte 0x88 /* DW_CFA_offset, column 0x8 */
|
||||
.uleb128 1
|
||||
.align 4
|
||||
.LENDCIEDLSI:
|
||||
.long .LENDFDEDLSI-.LSTARTFDEDLSI /* Length FDE */
|
||||
.LSTARTFDEDLSI:
|
||||
.long .LSTARTFDEDLSI-.LSTARTFRAMEDLSI /* CIE pointer */
|
||||
.long .LSTART_vsyscall-. /* PC-relative start address */
|
||||
.long .LEND_vsyscall-.LSTART_vsyscall
|
||||
.uleb128 0
|
||||
/* What follows are the instructions for the table generation.
|
||||
We have to record all changes of the stack pointer. */
|
||||
.byte 0x40 + (.Lpush_ecx-.LSTART_vsyscall) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x08 /* RA at offset 8 now */
|
||||
.byte 0x40 + (.Lpush_edx-.Lpush_ecx) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x0c /* RA at offset 12 now */
|
||||
.byte 0x40 + (.Lenter_kernel-.Lpush_edx) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x10 /* RA at offset 16 now */
|
||||
.byte 0x85, 0x04 /* DW_CFA_offset %ebp -16 */
|
||||
/* Finally the epilogue. */
|
||||
.byte 0x40 + (.Lpop_ebp-.Lenter_kernel) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x0c /* RA at offset 12 now */
|
||||
.byte 0xc5 /* DW_CFA_restore %ebp */
|
||||
.byte 0x40 + (.Lpop_edx-.Lpop_ebp) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x08 /* RA at offset 8 now */
|
||||
.byte 0x40 + (.Lpop_ecx-.Lpop_edx) /* DW_CFA_advance_loc */
|
||||
.byte 0x0e /* DW_CFA_def_cfa_offset */
|
||||
.byte 0x04 /* RA at offset 4 now */
|
||||
.align 4
|
||||
.LENDFDEDLSI:
|
||||
.previous
|
||||
|
||||
/*
|
||||
* Emit a symbol with the size of this .eh_frame data,
|
||||
* to verify it matches the other versions.
|
||||
*/
|
||||
VDSO32_vsyscall_eh_frame_size = (.LENDFDEDLSI-.LSTARTFRAMEDLSI)
|
||||
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Code for the vDSO. This version uses the old int $0x80 method.
|
||||
*/
|
||||
|
||||
#include <asm/dwarf2.h>
|
||||
#include <asm/cpufeature.h>
|
||||
#include <asm/alternative-asm.h>
|
||||
|
||||
/*
|
||||
* First get the common code for the sigreturn entry points.
|
||||
* This must come first.
|
||||
*/
|
||||
#include "sigreturn.S"
|
||||
|
||||
.text
|
||||
.globl __kernel_vsyscall
|
||||
.type __kernel_vsyscall,@function
|
||||
ALIGN
|
||||
__kernel_vsyscall:
|
||||
CFI_STARTPROC
|
||||
/*
|
||||
* Reshuffle regs so that all of any of the entry instructions
|
||||
* will preserve enough state.
|
||||
*/
|
||||
pushl %edx
|
||||
CFI_ADJUST_CFA_OFFSET 4
|
||||
CFI_REL_OFFSET edx, 0
|
||||
pushl %ecx
|
||||
CFI_ADJUST_CFA_OFFSET 4
|
||||
CFI_REL_OFFSET ecx, 0
|
||||
movl %esp, %ecx
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/* If SYSENTER (Intel) or SYSCALL32 (AMD) is available, use it. */
|
||||
ALTERNATIVE_2 "", "sysenter", X86_FEATURE_SYSENTER32, \
|
||||
"syscall", X86_FEATURE_SYSCALL32
|
||||
#else
|
||||
ALTERNATIVE "", "sysenter", X86_FEATURE_SEP
|
||||
#endif
|
||||
|
||||
/* Enter using int $0x80 */
|
||||
movl (%esp), %ecx
|
||||
int $0x80
|
||||
GLOBAL(int80_landing_pad)
|
||||
|
||||
/* Restore ECX and EDX in case they were clobbered. */
|
||||
popl %ecx
|
||||
CFI_RESTORE ecx
|
||||
CFI_ADJUST_CFA_OFFSET -4
|
||||
popl %edx
|
||||
CFI_RESTORE edx
|
||||
CFI_ADJUST_CFA_OFFSET -4
|
||||
ret
|
||||
CFI_ENDPROC
|
||||
|
||||
.size __kernel_vsyscall,.-__kernel_vsyscall
|
||||
.previous
|
||||
@@ -180,21 +180,10 @@ up_fail:
|
||||
#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
|
||||
static int load_vdso32(void)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (vdso32_enabled != 1) /* Other values all mean "disabled" */
|
||||
return 0;
|
||||
|
||||
ret = map_vdso(selected_vdso32, false);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (selected_vdso32->sym_VDSO32_SYSENTER_RETURN)
|
||||
current_thread_info()->sysenter_return =
|
||||
current->mm->context.vdso +
|
||||
selected_vdso32->sym_VDSO32_SYSENTER_RETURN;
|
||||
|
||||
return 0;
|
||||
return map_vdso(&vdso_image_32, false);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -38,7 +38,14 @@
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include "vsyscall_trace.h"
|
||||
|
||||
static enum { EMULATE, NATIVE, NONE } vsyscall_mode = EMULATE;
|
||||
static enum { EMULATE, NATIVE, NONE } vsyscall_mode =
|
||||
#if defined(CONFIG_LEGACY_VSYSCALL_NATIVE)
|
||||
NATIVE;
|
||||
#elif defined(CONFIG_LEGACY_VSYSCALL_NONE)
|
||||
NONE;
|
||||
#else
|
||||
EMULATE;
|
||||
#endif
|
||||
|
||||
static int __init vsyscall_setup(char *str)
|
||||
{
|
||||
|
||||
@@ -289,7 +289,7 @@ int ia32_setup_frame(int sig, struct ksignal *ksig,
|
||||
/* Return stub is in 32bit vsyscall page */
|
||||
if (current->mm->context.vdso)
|
||||
restorer = current->mm->context.vdso +
|
||||
selected_vdso32->sym___kernel_sigreturn;
|
||||
vdso_image_32.sym___kernel_sigreturn;
|
||||
else
|
||||
restorer = &frame->retcode;
|
||||
}
|
||||
@@ -368,7 +368,7 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig,
|
||||
restorer = ksig->ka.sa.sa_restorer;
|
||||
else
|
||||
restorer = current->mm->context.vdso +
|
||||
selected_vdso32->sym___kernel_rt_sigreturn;
|
||||
vdso_image_32.sym___kernel_rt_sigreturn;
|
||||
put_user_ex(ptr_to_compat(restorer), &frame->pretcode);
|
||||
|
||||
/*
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#ifndef _ASM_X86_DWARF2_H
|
||||
#define _ASM_X86_DWARF2_H
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#warning "asm/dwarf2.h should be only included in pure assembly files"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Macros for dwarf2 CFI unwind table entries.
|
||||
* See "as.info" for details on these pseudo ops. Unfortunately
|
||||
* they are only supported in very new binutils, so define them
|
||||
* away for older version.
|
||||
*/
|
||||
|
||||
#ifdef CONFIG_AS_CFI
|
||||
|
||||
#define CFI_STARTPROC .cfi_startproc
|
||||
#define CFI_ENDPROC .cfi_endproc
|
||||
#define CFI_DEF_CFA .cfi_def_cfa
|
||||
#define CFI_DEF_CFA_REGISTER .cfi_def_cfa_register
|
||||
#define CFI_DEF_CFA_OFFSET .cfi_def_cfa_offset
|
||||
#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset
|
||||
#define CFI_OFFSET .cfi_offset
|
||||
#define CFI_REL_OFFSET .cfi_rel_offset
|
||||
#define CFI_REGISTER .cfi_register
|
||||
#define CFI_RESTORE .cfi_restore
|
||||
#define CFI_REMEMBER_STATE .cfi_remember_state
|
||||
#define CFI_RESTORE_STATE .cfi_restore_state
|
||||
#define CFI_UNDEFINED .cfi_undefined
|
||||
#define CFI_ESCAPE .cfi_escape
|
||||
|
||||
#ifdef CONFIG_AS_CFI_SIGNAL_FRAME
|
||||
#define CFI_SIGNAL_FRAME .cfi_signal_frame
|
||||
#else
|
||||
#define CFI_SIGNAL_FRAME
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_AS_CFI_SECTIONS) && defined(__ASSEMBLY__)
|
||||
#ifndef BUILD_VDSO
|
||||
/*
|
||||
* Emit CFI data in .debug_frame sections, not .eh_frame sections.
|
||||
* The latter we currently just discard since we don't do DWARF
|
||||
* unwinding at runtime. So only the offline DWARF information is
|
||||
* useful to anyone. Note we should not use this directive if
|
||||
* vmlinux.lds.S gets changed so it doesn't discard .eh_frame.
|
||||
*/
|
||||
.cfi_sections .debug_frame
|
||||
#else
|
||||
/*
|
||||
* For the vDSO, emit both runtime unwind information and debug
|
||||
* symbols for the .dbg file.
|
||||
*/
|
||||
.cfi_sections .eh_frame, .debug_frame
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#else
|
||||
|
||||
/*
|
||||
* Due to the structure of pre-exisiting code, don't use assembler line
|
||||
* comment character # to ignore the arguments. Instead, use a dummy macro.
|
||||
*/
|
||||
.macro cfi_ignore a=0, b=0, c=0, d=0
|
||||
.endm
|
||||
|
||||
#define CFI_STARTPROC cfi_ignore
|
||||
#define CFI_ENDPROC cfi_ignore
|
||||
#define CFI_DEF_CFA cfi_ignore
|
||||
#define CFI_DEF_CFA_REGISTER cfi_ignore
|
||||
#define CFI_DEF_CFA_OFFSET cfi_ignore
|
||||
#define CFI_ADJUST_CFA_OFFSET cfi_ignore
|
||||
#define CFI_OFFSET cfi_ignore
|
||||
#define CFI_REL_OFFSET cfi_ignore
|
||||
#define CFI_REGISTER cfi_ignore
|
||||
#define CFI_RESTORE cfi_ignore
|
||||
#define CFI_REMEMBER_STATE cfi_ignore
|
||||
#define CFI_RESTORE_STATE cfi_ignore
|
||||
#define CFI_UNDEFINED cfi_ignore
|
||||
#define CFI_ESCAPE cfi_ignore
|
||||
#define CFI_SIGNAL_FRAME cfi_ignore
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* _ASM_X86_DWARF2_H */
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user