Merge branch 'x86-extable-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull exception table generation updates from Ingo Molnar:
 "The biggest change here is to allow the build-time sorting of the
  exception table, to speed up booting.  This is achieved by the
  architecture enabling BUILDTIME_EXTABLE_SORT.  This option is enabled
  for x86 and MIPS currently.

  On x86 a number of fixes and changes were needed to allow build-time
  sorting of the exception table, in particular a relocation invariant
  exception table format was needed.  This required the abstracting out
  of exception table protocol and the removal of 20 years of accumulated
  assumptions about the x86 exception table format.

  While at it, this tree also cleans up various other aspects of
  exception handling, such as early(er) exception handling for
  rdmsr_safe() et al.

  All in one, as the result of these changes the x86 exception code is
  now pretty nice and modern.  As an added bonus any regressions in this
  code will be early and violent crashes, so if you see any of those,
  you'll know whom to blame!"

Fix up trivial conflicts in arch/{mips,x86}/Kconfig files due to nearby
modifications of other core architecture options.

* 'x86-extable-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (35 commits)
  Revert "x86, extable: Disable presorted exception table for now"
  scripts/sortextable: Handle relative entries, and other cleanups
  x86, extable: Switch to relative exception table entries
  x86, extable: Disable presorted exception table for now
  x86, extable: Add _ASM_EXTABLE_EX() macro
  x86, extable: Remove open-coded exception table entries in arch/x86/ia32/ia32entry.S
  x86, extable: Remove open-coded exception table entries in arch/x86/include/asm/xsave.h
  x86, extable: Remove open-coded exception table entries in arch/x86/include/asm/kvm_host.h
  x86, extable: Remove the now-unused __ASM_EX_SEC macros
  x86, extable: Remove open-coded exception table entries in arch/x86/xen/xen-asm_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/um/checksum_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/usercopy_32.c
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/putuser.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/getuser.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/csum-copy_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/copy_user_nocache_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/copy_user_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/checksum_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/kernel/test_rodata.c
  x86, extable: Remove open-coded exception table entries in arch/x86/kernel/entry_64.S
  ...

Makefile

@@ -796,6 +796,10 @@ quiet_cmd_vmlinux_version = GEN     .version
 quiet_cmd_sysmap = SYSMAP
       cmd_sysmap = $(CONFIG_SHELL) $(srctree)/scripts/mksysmap
 
+# Sort exception table at build time
+quiet_cmd_sortextable = SORTEX
+      cmd_sortextable = $(objtree)/scripts/sortextable
+
 # Link of vmlinux
 # If CONFIG_KALLSYMS is set .version is already updated
 # Generate System.map and verify that the content is consistent
@@ -808,6 +812,12 @@ define rule_vmlinux__
 	$(call cmd,vmlinux__)
 	$(Q)echo 'cmd_$@ := $(cmd_vmlinux__)' > $(@D)/.$(@F).cmd
 
+	$(if $(CONFIG_BUILDTIME_EXTABLE_SORT),				\
+	  $(Q)$(if $($(quiet)cmd_sortextable),				\
+	    echo '  $($(quiet)cmd_sortextable)  vmlinux' &&)		\
+	  $(cmd_sortextable)  vmlinux)
+
+
 	$(Q)$(if $($(quiet)cmd_sysmap),                                      \
 	  echo '  $($(quiet)cmd_sysmap)  System.map' &&)                     \
 	$(cmd_sysmap) $@ System.map;                                         \

arch/mips/Kconfig

@@ -30,6 +30,7 @@ config MIPS
 	select HAVE_MEMBLOCK_NODE_MAP
 	select ARCH_DISCARD_MEMBLOCK
 	select GENERIC_SMP_IDLE_THREAD
+	select BUILDTIME_EXTABLE_SORT
 
 menu "Machine selection"
 

arch/x86/Kconfig

@@ -83,6 +83,7 @@ config X86
 	select DCACHE_WORD_ACCESS
 	select GENERIC_SMP_IDLE_THREAD
 	select HAVE_ARCH_SECCOMP_FILTER
+	select BUILDTIME_EXTABLE_SORT
 
 config INSTRUCTION_DECODER
 	def_bool (KPROBES || PERF_EVENTS)

arch/x86/ia32/ia32entry.S

@@ -13,6 +13,7 @@
 #include <asm/thread_info.h>	
 #include <asm/segment.h>
 #include <asm/irqflags.h>
+#include <asm/asm.h>
 #include <linux/linkage.h>
 #include <linux/err.h>
 
@@ -146,9 +147,7 @@ ENTRY(ia32_sysenter_target)
  	/* no need to do an access_ok check here because rbp has been
  	   32bit zero extended */ 
 1:	movl	(%rbp),%ebp
- 	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,ia32_badarg)
- 	.quad 1b,ia32_badarg
- 	.previous	
 	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
 	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
 	CFI_REMEMBER_STATE
@@ -303,9 +302,7 @@ ENTRY(ia32_cstar_target)
 	   32bit zero extended */ 
 	/* hardware stack frame is complete now */	
 1:	movl	(%r8),%r9d
-	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,ia32_badarg)
-	.quad 1b,ia32_badarg
-	.previous	
 	orl     $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
 	testl   $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
 	CFI_REMEMBER_STATE

arch/x86/include/asm/asm.h

@@ -4,11 +4,9 @@
 #ifdef __ASSEMBLY__
 # define __ASM_FORM(x)	x
 # define __ASM_FORM_COMMA(x) x,
-# define __ASM_EX_SEC	.section __ex_table, "a"
 #else
 # define __ASM_FORM(x)	" " #x " "
 # define __ASM_FORM_COMMA(x) " " #x ","
-# define __ASM_EX_SEC	" .section __ex_table,\"a\"\n"
 #endif
 
 #ifdef CONFIG_X86_32
@@ -42,17 +40,33 @@
 
 /* Exception table entry */
 #ifdef __ASSEMBLY__
-# define _ASM_EXTABLE(from,to)	    \
+# define _ASM_EXTABLE(from,to)					\
-	__ASM_EX_SEC ;		    \
+	.pushsection "__ex_table","a" ;				\
-	_ASM_ALIGN ;		    \
+	.balign 8 ;						\
-	_ASM_PTR from , to ;	    \
+	.long (from) - . ;					\
-	.previous
+	.long (to) - . ;					\
+	.popsection
+
+# define _ASM_EXTABLE_EX(from,to)				\
+	.pushsection "__ex_table","a" ;				\
+	.balign 8 ;						\
+	.long (from) - . ;					\
+	.long (to) - . + 0x7ffffff0 ;				\
+	.popsection
 #else
-# define _ASM_EXTABLE(from,to) \
+# define _ASM_EXTABLE(from,to)					\
-	__ASM_EX_SEC	\
+	" .pushsection \"__ex_table\",\"a\"\n"			\
-	_ASM_ALIGN "\n" \
+	" .balign 8\n"						\
-	_ASM_PTR #from "," #to "\n" \
+	" .long (" #from ") - .\n"				\
-	" .previous\n"
+	" .long (" #to ") - .\n"				\
+	" .popsection\n"
+
+# define _ASM_EXTABLE_EX(from,to)				\
+	" .pushsection \"__ex_table\",\"a\"\n"			\
+	" .balign 8\n"						\
+	" .long (" #from ") - .\n"				\
+	" .long (" #to ") - . + 0x7ffffff0\n"			\
+	" .popsection\n"
 #endif
 
 #endif /* _ASM_X86_ASM_H */

arch/x86/include/asm/kvm_host.h

@@ -27,6 +27,7 @@
 #include <asm/desc.h>
 #include <asm/mtrr.h>
 #include <asm/msr-index.h>
+#include <asm/asm.h>
 
 #define KVM_MAX_VCPUS 254
 #define KVM_SOFT_MAX_VCPUS 160
@@ -921,9 +922,7 @@ extern bool kvm_rebooting;
 	__ASM_SIZE(push) " $666b \n\t"	      \
 	"call kvm_spurious_fault \n\t"	      \
 	".popsection \n\t" \
-	".pushsection __ex_table, \"a\" \n\t" \
+	_ASM_EXTABLE(666b, 667b)
-	_ASM_PTR " 666b, 667b \n\t" \
-	".popsection"
 
 #define __kvm_handle_fault_on_reboot(insn)		\
 	____kvm_handle_fault_on_reboot(insn, "")

arch/x86/include/asm/msr.h

@@ -169,14 +169,7 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high)
 	return native_write_msr_safe(msr, low, high);
 }
 
-/*
+/* rdmsr with exception handling */
- * rdmsr with exception handling.
- *
- * Please note that the exception handling works only after we've
- * switched to the "smart" #GP handler in trap_init() which knows about
- * exception tables - using this macro earlier than that causes machine
- * hangs on boxes which do not implement the @msr in the first argument.
- */
 #define rdmsr_safe(msr, p1, p2)					\
 ({								\
 	int __err;						\

arch/x86/include/asm/nops.h

@@ -87,7 +87,11 @@
 #define P6_NOP8	0x0f,0x1f,0x84,0x00,0,0,0,0
 #define P6_NOP5_ATOMIC P6_NOP5
 
+#ifdef __ASSEMBLY__
+#define _ASM_MK_NOP(x) .byte x
+#else
 #define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
+#endif
 
 #if defined(CONFIG_MK7)
 #define ASM_NOP1 _ASM_MK_NOP(K7_NOP1)

arch/x86/include/asm/paravirt.h

@@ -1023,10 +1023,8 @@ extern void default_banner(void);
 		  call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)		\
 		 )
 
-#define GET_CR2_INTO_RCX				\
+#define GET_CR2_INTO_RAX				\
-	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2);	\
+	call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
-	movq %rax, %rcx;				\
-	xorq %rax, %rax;
 
 #define PARAVIRT_ADJUST_EXCEPTION_FRAME					\
 	PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \

arch/x86/include/asm/segment.h

@@ -205,13 +205,15 @@
 
 #define IDT_ENTRIES 256
 #define NUM_EXCEPTION_VECTORS 32
+/* Bitmask of exception vectors which push an error code on the stack */
+#define EXCEPTION_ERRCODE_MASK  0x00027d00
 #define GDT_SIZE (GDT_ENTRIES * 8)
 #define GDT_ENTRY_TLS_ENTRIES 3
 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES * 8)
 
 #ifdef __KERNEL__
 #ifndef __ASSEMBLY__
-extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][10];
+extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][2+2+5];
 
 /*
  * Load a segment. Fall back on loading the zero

arch/x86/include/asm/uaccess.h

@@ -79,11 +79,12 @@
 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
 
 /*
- * The exception table consists of pairs of addresses: the first is the
+ * The exception table consists of pairs of addresses relative to the
- * address of an instruction that is allowed to fault, and the second is
+ * exception table enty itself: the first is the address of an
- * the address at which the program should continue.  No registers are
+ * instruction that is allowed to fault, and the second is the address
- * modified, so it is entirely up to the continuation code to figure out
+ * at which the program should continue.  No registers are modified,
- * what to do.
+ * so it is entirely up to the continuation code to figure out what to
+ * do.
  *
  * All the routines below use bits of fixup code that are out of line
  * with the main instruction path.  This means when everything is well,
@@ -92,10 +93,14 @@
  */
 
 struct exception_table_entry {
-	unsigned long insn, fixup;
+	int insn, fixup;
 };
+/* This is not the generic standard exception_table_entry format */
+#define ARCH_HAS_SORT_EXTABLE
+#define ARCH_HAS_SEARCH_EXTABLE
 
 extern int fixup_exception(struct pt_regs *regs);
+extern int early_fixup_exception(unsigned long *ip);
 
 /*
  * These are the main single-value transfer routines.  They automatically
@@ -202,8 +207,8 @@ extern int __get_user_bad(void);
 	asm volatile("1:	movl %%eax,0(%1)\n"			\
 		     "2:	movl %%edx,4(%1)\n"			\
 		     "3:\n"						\
-		     _ASM_EXTABLE(1b, 2b - 1b)				\
+		     _ASM_EXTABLE_EX(1b, 2b)				\
-		     _ASM_EXTABLE(2b, 3b - 2b)				\
+		     _ASM_EXTABLE_EX(2b, 3b)				\
 		     : : "A" (x), "r" (addr))
 
 #define __put_user_x8(x, ptr, __ret_pu)				\
@@ -408,7 +413,7 @@ do {									\
 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)			\
 	asm volatile("1:	mov"itype" %1,%"rtype"0\n"		\
 		     "2:\n"						\
-		     _ASM_EXTABLE(1b, 2b - 1b)				\
+		     _ASM_EXTABLE_EX(1b, 2b)				\
 		     : ltype(x) : "m" (__m(addr)))
 
 #define __put_user_nocheck(x, ptr, size)			\
@@ -450,7 +455,7 @@ struct __large_struct { unsigned long buf[100]; };
 #define __put_user_asm_ex(x, addr, itype, rtype, ltype)			\
 	asm volatile("1:	mov"itype" %"rtype"0,%1\n"		\
 		     "2:\n"						\
-		     _ASM_EXTABLE(1b, 2b - 1b)				\
+		     _ASM_EXTABLE_EX(1b, 2b)				\
 		     : : ltype(x), "m" (__m(addr)))
 
 /*

arch/x86/include/asm/xsave.h

@@ -80,10 +80,7 @@ static inline int xsave_user(struct xsave_struct __user *buf)
 			     "3:  movl $-1,%[err]\n"
 			     "    jmp  2b\n"
 			     ".previous\n"
-			     ".section __ex_table,\"a\"\n"
+			     _ASM_EXTABLE(1b,3b)
-			     _ASM_ALIGN "\n"
-			     _ASM_PTR "1b,3b\n"
-			     ".previous"
 			     : [err] "=r" (err)
 			     : "D" (buf), "a" (-1), "d" (-1), "0" (0)
 			     : "memory");
@@ -106,10 +103,7 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
 			     "3:  movl $-1,%[err]\n"
 			     "    jmp  2b\n"
 			     ".previous\n"
-			     ".section __ex_table,\"a\"\n"
+			     _ASM_EXTABLE(1b,3b)
-			     _ASM_ALIGN "\n"
-			     _ASM_PTR "1b,3b\n"
-			     ".previous"
 			     : [err] "=r" (err)
 			     : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
 			     : "memory");	/* memory required? */

arch/x86/kernel/entry_32.S

@@ -56,6 +56,7 @@
 #include <asm/irq_vectors.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative-asm.h>
+#include <asm/asm.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
 #include <linux/elf-em.h>
@@ -151,10 +152,8 @@
 .pushsection .fixup, "ax"
 99:	movl $0, (%esp)
 	jmp 98b
-.section __ex_table, "a"
-	.align 4
-	.long 98b, 99b
 .popsection
+	_ASM_EXTABLE(98b,99b)
 .endm
 
 .macro PTGS_TO_GS
@@ -164,10 +163,8 @@
 .pushsection .fixup, "ax"
 99:	movl $0, PT_GS(%esp)
 	jmp 98b
-.section __ex_table, "a"
-	.align 4
-	.long 98b, 99b
 .popsection
+	_ASM_EXTABLE(98b,99b)
 .endm
 
 .macro GS_TO_REG reg
@@ -249,12 +246,10 @@
 	jmp 2b
 6:	movl $0, (%esp)
 	jmp 3b
-.section __ex_table, "a"
-	.align 4
-	.long 1b, 4b
-	.long 2b, 5b
-	.long 3b, 6b
 .popsection
+	_ASM_EXTABLE(1b,4b)
+	_ASM_EXTABLE(2b,5b)
+	_ASM_EXTABLE(3b,6b)
 	POP_GS_EX
 .endm
 
@@ -415,10 +410,7 @@ sysenter_past_esp:
 	jae syscall_fault
 1:	movl (%ebp),%ebp
 	movl %ebp,PT_EBP(%esp)
-.section __ex_table,"a"
+	_ASM_EXTABLE(1b,syscall_fault)
-	.align 4
-	.long 1b,syscall_fault
-.previous
 
 	GET_THREAD_INFO(%ebp)
 
@@ -485,10 +477,8 @@ sysexit_audit:
 .pushsection .fixup,"ax"
 2:	movl $0,PT_FS(%esp)
 	jmp 1b
-.section __ex_table,"a"
-	.align 4
-	.long 1b,2b
 .popsection
+	_ASM_EXTABLE(1b,2b)
 	PTGS_TO_GS_EX
 ENDPROC(ia32_sysenter_target)
 
@@ -543,10 +533,7 @@ ENTRY(iret_exc)
 	pushl $do_iret_error
 	jmp error_code
 .previous
-.section __ex_table,"a"
+	_ASM_EXTABLE(irq_return,iret_exc)
-	.align 4
-	.long irq_return,iret_exc
-.previous
 
 	CFI_RESTORE_STATE
 ldt_ss:
@@ -901,10 +888,7 @@ END(device_not_available)
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
 	iret
-.section __ex_table,"a"
+	_ASM_EXTABLE(native_iret, iret_exc)
-	.align 4
-	.long native_iret, iret_exc
-.previous
 END(native_iret)
 
 ENTRY(native_irq_enable_sysexit)
@@ -1093,13 +1077,10 @@ ENTRY(xen_failsafe_callback)
 	movl %eax,16(%esp)
 	jmp 4b
 .previous
-.section __ex_table,"a"
+	_ASM_EXTABLE(1b,6b)
-	.align 4
+	_ASM_EXTABLE(2b,7b)
-	.long 1b,6b
+	_ASM_EXTABLE(3b,8b)
-	.long 2b,7b
+	_ASM_EXTABLE(4b,9b)
-	.long 3b,8b
-	.long 4b,9b
-.previous
 ENDPROC(xen_failsafe_callback)
 
 BUILD_INTERRUPT3(xen_hvm_callback_vector, XEN_HVM_EVTCHN_CALLBACK,

arch/x86/kernel/entry_64.S

@@ -55,6 +55,7 @@
 #include <asm/paravirt.h>
 #include <asm/ftrace.h>
 #include <asm/percpu.h>
+#include <asm/asm.h>
 #include <linux/err.h>
 
 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this.  */
@@ -900,18 +901,12 @@ restore_args:
 
 irq_return:
 	INTERRUPT_RETURN
-
+	_ASM_EXTABLE(irq_return, bad_iret)
-	.section __ex_table, "a"
-	.quad irq_return, bad_iret
-	.previous
 
 #ifdef CONFIG_PARAVIRT
 ENTRY(native_iret)
 	iretq
-
+	_ASM_EXTABLE(native_iret, bad_iret)
-	.section __ex_table,"a"
-	.quad native_iret, bad_iret
-	.previous
 #endif
 
 	.section .fixup,"ax"
@@ -1181,10 +1176,7 @@ gs_change:
 	CFI_ENDPROC
 END(native_load_gs_index)
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(gs_change,bad_gs)
-	.align 8
-	.quad gs_change,bad_gs
-	.previous
 	.section .fixup,"ax"
 	/* running with kernelgs */
 bad_gs:

arch/x86/kernel/head_32.S

@@ -21,6 +21,7 @@
 #include <asm/msr-index.h>
 #include <asm/cpufeature.h>
 #include <asm/percpu.h>
+#include <asm/nops.h>
 
 /* Physical address */
 #define pa(X) ((X) - __PAGE_OFFSET)
@@ -363,28 +364,23 @@ default_entry:
 	pushl $0
 	popfl
 
-#ifdef CONFIG_SMP
-	cmpb $0, ready
-	jnz checkCPUtype
-#endif /* CONFIG_SMP */
-
 /*
  * start system 32-bit setup. We need to re-do some of the things done
  * in 16-bit mode for the "real" operations.
  */
-	call setup_idt
+	movl setup_once_ref,%eax
-
+	andl %eax,%eax
-checkCPUtype:
+	jz 1f				# Did we do this already?
-
+	call *%eax
-	movl $-1,X86_CPUID		#  -1 for no CPUID initially
+1:
-
+	
 /* check if it is 486 or 386. */
 /*
  * XXX - this does a lot of unnecessary setup.  Alignment checks don't
  * apply at our cpl of 0 and the stack ought to be aligned already, and
  * we don't need to preserve eflags.
  */
-
+	movl $-1,X86_CPUID	# -1 for no CPUID initially
 	movb $3,X86		# at least 386
 	pushfl			# push EFLAGS
 	popl %eax		# get EFLAGS
@@ -450,21 +446,6 @@ is386:	movl $2,%ecx		# set MP
 	movl $(__KERNEL_PERCPU), %eax
 	movl %eax,%fs			# set this cpu's percpu
 
-#ifdef CONFIG_CC_STACKPROTECTOR
-	/*
-	 * The linker can't handle this by relocation.  Manually set
-	 * base address in stack canary segment descriptor.
-	 */
-	cmpb $0,ready
-	jne 1f
-	movl $gdt_page,%eax
-	movl $stack_canary,%ecx
-	movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
-	shrl $16, %ecx
-	movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
-	movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax)
-1:
-#endif
 	movl $(__KERNEL_STACK_CANARY),%eax
 	movl %eax,%gs
 
@@ -473,7 +454,6 @@ is386:	movl $2,%ecx		# set MP
 
 	cld			# gcc2 wants the direction flag cleared at all times
 	pushl $0		# fake return address for unwinder
-	movb $1, ready
 	jmp *(initial_code)
 
 /*
@@ -495,81 +475,122 @@ check_x87:
 	.byte 0xDB,0xE4		/* fsetpm for 287, ignored by 387 */
 	ret
 
+	
+#include "verify_cpu.S"
+
 /*
- *  setup_idt
+ *  setup_once
  *
- *  sets up a idt with 256 entries pointing to
+ *  The setup work we only want to run on the BSP.
- *  ignore_int, interrupt gates. It doesn't actually load
- *  idt - that can be done only after paging has been enabled
- *  and the kernel moved to PAGE_OFFSET. Interrupts
- *  are enabled elsewhere, when we can be relatively
- *  sure everything is ok.
  *
  *  Warning: %esi is live across this function.
  */
-setup_idt:
+__INIT
-	lea ignore_int,%edx
+setup_once:
+	/*
+	 * Set up a idt with 256 entries pointing to ignore_int,
+	 * interrupt gates. It doesn't actually load idt - that needs
+	 * to be done on each CPU. Interrupts are enabled elsewhere,
+	 * when we can be relatively sure everything is ok.
+	 */
+
+	movl $idt_table,%edi
+	movl $early_idt_handlers,%eax
+	movl $NUM_EXCEPTION_VECTORS,%ecx
+1:
+	movl %eax,(%edi)
+	movl %eax,4(%edi)
+	/* interrupt gate, dpl=0, present */
+	movl $(0x8E000000 + __KERNEL_CS),2(%edi)
+	addl $9,%eax
+	addl $8,%edi
+	loop 1b
+
+	movl $256 - NUM_EXCEPTION_VECTORS,%ecx
+	movl $ignore_int,%edx
 	movl $(__KERNEL_CS << 16),%eax
 	movw %dx,%ax		/* selector = 0x0010 = cs */
 	movw $0x8E00,%dx	/* interrupt gate - dpl=0, present */
-
+2:
-	lea idt_table,%edi
-	mov $256,%ecx
-rp_sidt:
 	movl %eax,(%edi)
 	movl %edx,4(%edi)
 	addl $8,%edi
-	dec %ecx
+	loop 2b
-	jne rp_sidt
 
-.macro	set_early_handler handler,trapno
+#ifdef CONFIG_CC_STACKPROTECTOR
-	lea \handler,%edx
+	/*
-	movl $(__KERNEL_CS << 16),%eax
+	 * Configure the stack canary. The linker can't handle this by
-	movw %dx,%ax
+	 * relocation.  Manually set base address in stack canary
-	movw $0x8E00,%dx	/* interrupt gate - dpl=0, present */
+	 * segment descriptor.
-	lea idt_table,%edi
+	 */
-	movl %eax,8*\trapno(%edi)
+	movl $gdt_page,%eax
-	movl %edx,8*\trapno+4(%edi)
+	movl $stack_canary,%ecx
-.endm
+	movw %cx, 8 * GDT_ENTRY_STACK_CANARY + 2(%eax)
-
+	shrl $16, %ecx
-	set_early_handler handler=early_divide_err,trapno=0
+	movb %cl, 8 * GDT_ENTRY_STACK_CANARY + 4(%eax)
-	set_early_handler handler=early_illegal_opcode,trapno=6
+	movb %ch, 8 * GDT_ENTRY_STACK_CANARY + 7(%eax)
-	set_early_handler handler=early_protection_fault,trapno=13
+#endif
-	set_early_handler handler=early_page_fault,trapno=14
 
+	andl $0,setup_once_ref	/* Once is enough, thanks */
 	ret
 
-early_divide_err:
+ENTRY(early_idt_handlers)
-	xor %edx,%edx
+	# 36(%esp) %eflags
-	pushl $0	/* fake errcode */
+	# 32(%esp) %cs
-	jmp early_fault
+	# 28(%esp) %eip
-
+	# 24(%rsp) error code
-early_illegal_opcode:
+	i = 0
-	movl $6,%edx
+	.rept NUM_EXCEPTION_VECTORS
-	pushl $0	/* fake errcode */
+	.if (EXCEPTION_ERRCODE_MASK >> i) & 1
-	jmp early_fault
+	ASM_NOP2
-
+	.else
-early_protection_fault:
+	pushl $0		# Dummy error code, to make stack frame uniform
-	movl $13,%edx
+	.endif
-	jmp early_fault
+	pushl $i		# 20(%esp) Vector number
-
+	jmp early_idt_handler
-early_page_fault:
+	i = i + 1
-	movl $14,%edx
+	.endr
-	jmp early_fault
+ENDPROC(early_idt_handlers)
-
+	
-early_fault:
+	/* This is global to keep gas from relaxing the jumps */
+ENTRY(early_idt_handler)
 	cld
-#ifdef CONFIG_PRINTK
+	cmpl $2,%ss:early_recursion_flag
-	pusha
+	je hlt_loop
+	incl %ss:early_recursion_flag
+
+	push %eax		# 16(%esp)
+	push %ecx		# 12(%esp)
+	push %edx		#  8(%esp)
+	push %ds		#  4(%esp)
+	push %es		#  0(%esp)
 	movl $(__KERNEL_DS),%eax
 	movl %eax,%ds
 	movl %eax,%es
-	cmpl $2,early_recursion_flag
+
-	je hlt_loop
+	cmpl $(__KERNEL_CS),32(%esp)
-	incl early_recursion_flag
+	jne 10f
+
+	leal 28(%esp),%eax	# Pointer to %eip
+	call early_fixup_exception
+	andl %eax,%eax
+	jnz ex_entry		/* found an exception entry */
+
+10:
+#ifdef CONFIG_PRINTK
+	xorl %eax,%eax
+	movw %ax,2(%esp)	/* clean up the segment values on some cpus */
+	movw %ax,6(%esp)
+	movw %ax,34(%esp)
+	leal  40(%esp),%eax
+	pushl %eax		/* %esp before the exception */
+	pushl %ebx
+	pushl %ebp
+	pushl %esi
+	pushl %edi
 	movl %cr2,%eax
 	pushl %eax
-	pushl %edx		/* trapno */
+	pushl (20+6*4)(%esp)	/* trapno */
 	pushl $fault_msg
 	call printk
 #endif
@@ -578,6 +599,17 @@ hlt_loop:
 	hlt
 	jmp hlt_loop
 
+ex_entry:
+	pop %es
+	pop %ds
+	pop %edx
+	pop %ecx
+	pop %eax
+	addl $8,%esp		/* drop vector number and error code */
+	decl %ss:early_recursion_flag
+	iret
+ENDPROC(early_idt_handler)
+
 /* This is the default interrupt "handler" :-) */
 	ALIGN
 ignore_int:
@@ -611,13 +643,18 @@ ignore_int:
 	popl %eax
 #endif
 	iret
+ENDPROC(ignore_int)
+__INITDATA
+	.align 4
+early_recursion_flag:
+	.long 0
 
-#include "verify_cpu.S"
+__REFDATA
-
+	.align 4
-	__REFDATA
-.align 4
 ENTRY(initial_code)
 	.long i386_start_kernel
+ENTRY(setup_once_ref)
+	.long setup_once
 
 /*
  * BSS section
@@ -670,22 +707,19 @@ ENTRY(initial_page_table)
 ENTRY(stack_start)
 	.long init_thread_union+THREAD_SIZE
 
-early_recursion_flag:
+__INITRODATA
-	.long 0
-
-ready:	.byte 0
-
 int_msg:
 	.asciz "Unknown interrupt or fault at: %p %p %p\n"
 
 fault_msg:
 /* fault info: */
 	.ascii "BUG: Int %d: CR2 %p\n"
-/* pusha regs: */
+/* regs pushed in early_idt_handler: */
-	.ascii "     EDI %p  ESI %p  EBP %p  ESP %p\n"
+	.ascii "     EDI %p  ESI %p  EBP %p  EBX %p\n"
-	.ascii "     EBX %p  EDX %p  ECX %p  EAX %p\n"
+	.ascii "     ESP %p   ES %p   DS %p\n"
+	.ascii "     EDX %p  ECX %p  EAX %p\n"
 /* fault frame: */
-	.ascii "     err %p  EIP %p   CS %p  flg %p\n"
+	.ascii "     vec %p  err %p  EIP %p   CS %p  flg %p\n"
 	.ascii "Stack: %p %p %p %p %p %p %p %p\n"
 	.ascii "       %p %p %p %p %p %p %p %p\n"
 	.asciz "       %p %p %p %p %p %p %p %p\n"
@@ -699,6 +733,7 @@ fault_msg:
  * segment size, and 32-bit linear address value:
  */
 
+	.data
 .globl boot_gdt_descr
 .globl idt_descr
 

arch/x86/kernel/head_64.S

@@ -19,12 +19,15 @@
 #include <asm/cache.h>
 #include <asm/processor-flags.h>
 #include <asm/percpu.h>
+#include <asm/nops.h>
 
 #ifdef CONFIG_PARAVIRT
 #include <asm/asm-offsets.h>
 #include <asm/paravirt.h>
+#define GET_CR2_INTO(reg) GET_CR2_INTO_RAX ; movq %rax, reg
 #else
-#define GET_CR2_INTO_RCX movq %cr2, %rcx
+#define GET_CR2_INTO(reg) movq %cr2, reg
+#define INTERRUPT_RETURN iretq
 #endif
 
 /* we are not able to switch in one step to the final KERNEL ADDRESS SPACE
@@ -270,36 +273,56 @@ bad_address:
 	jmp bad_address
 
 	.section ".init.text","ax"
-#ifdef CONFIG_EARLY_PRINTK
 	.globl early_idt_handlers
 early_idt_handlers:
+	# 104(%rsp) %rflags
+	#  96(%rsp) %cs
+	#  88(%rsp) %rip
+	#  80(%rsp) error code
 	i = 0
 	.rept NUM_EXCEPTION_VECTORS
-	movl $i, %esi
+	.if (EXCEPTION_ERRCODE_MASK >> i) & 1
+	ASM_NOP2
+	.else
+	pushq $0		# Dummy error code, to make stack frame uniform
+	.endif
+	pushq $i		# 72(%rsp) Vector number
 	jmp early_idt_handler
 	i = i + 1
 	.endr
-#endif
 
 ENTRY(early_idt_handler)
-#ifdef CONFIG_EARLY_PRINTK
+	cld
+
 	cmpl $2,early_recursion_flag(%rip)
 	jz  1f
 	incl early_recursion_flag(%rip)
-	GET_CR2_INTO_RCX
+
-	movq %rcx,%r9
+	pushq %rax		# 64(%rsp)
-	xorl %r8d,%r8d		# zero for error code
+	pushq %rcx		# 56(%rsp)
-	movl %esi,%ecx		# get vector number
+	pushq %rdx		# 48(%rsp)
-	# Test %ecx against mask of vectors that push error code.
+	pushq %rsi		# 40(%rsp)
-	cmpl $31,%ecx
+	pushq %rdi		# 32(%rsp)
-	ja 0f
+	pushq %r8		# 24(%rsp)
-	movl $1,%eax
+	pushq %r9		# 16(%rsp)
-	salq %cl,%rax
+	pushq %r10		#  8(%rsp)
-	testl $0x27d00,%eax
+	pushq %r11		#  0(%rsp)
-	je 0f
+
-	popq %r8		# get error code
+	cmpl $__KERNEL_CS,96(%rsp)
-0:	movq 0(%rsp),%rcx	# get ip
+	jne 10f
-	movq 8(%rsp),%rdx	# get cs
+
+	leaq 88(%rsp),%rdi	# Pointer to %rip
+	call early_fixup_exception
+	andl %eax,%eax
+	jnz 20f			# Found an exception entry
+
+10:
+#ifdef CONFIG_EARLY_PRINTK
+	GET_CR2_INTO(%r9)	# can clobber any volatile register if pv
+	movl 80(%rsp),%r8d	# error code
+	movl 72(%rsp),%esi	# vector number
+	movl 96(%rsp),%edx	# %cs
+	movq 88(%rsp),%rcx	# %rip
 	xorl %eax,%eax
 	leaq early_idt_msg(%rip),%rdi
 	call early_printk
@@ -308,17 +331,32 @@ ENTRY(early_idt_handler)
 	call dump_stack
 #ifdef CONFIG_KALLSYMS	
 	leaq early_idt_ripmsg(%rip),%rdi
-	movq 0(%rsp),%rsi	# get rip again
+	movq 40(%rsp),%rsi	# %rip again
 	call __print_symbol
 #endif
 #endif /* EARLY_PRINTK */
 1:	hlt
 	jmp 1b
 
-#ifdef CONFIG_EARLY_PRINTK
+20:	# Exception table entry found
+	popq %r11
+	popq %r10
+	popq %r9
+	popq %r8
+	popq %rdi
+	popq %rsi
+	popq %rdx
+	popq %rcx
+	popq %rax
+	addq $16,%rsp		# drop vector number and error code
+	decl early_recursion_flag(%rip)
+	INTERRUPT_RETURN
+
+	.balign 4
 early_recursion_flag:
 	.long 0
 
+#ifdef CONFIG_EARLY_PRINTK
 early_idt_msg:
 	.asciz "PANIC: early exception %02lx rip %lx:%lx error %lx cr2 %lx\n"
 early_idt_ripmsg:

arch/x86/kernel/test_rodata.c

@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <asm/cacheflush.h>
 #include <asm/sections.h>
+#include <asm/asm.h>
 
 int rodata_test(void)
 {
@@ -42,14 +43,7 @@ int rodata_test(void)
 		".section .fixup,\"ax\"\n"
 		"2:	jmp 1b\n"
 		".previous\n"
-		".section __ex_table,\"a\"\n"
+		_ASM_EXTABLE(0b,2b)
-		"       .align 16\n"
-#ifdef CONFIG_X86_32
-		"	.long 0b,2b\n"
-#else
-		"	.quad 0b,2b\n"
-#endif
-		".previous"
 		: [rslt] "=r" (result)
 		: [rodata_test] "r" (&rodata_test_data), [zero] "r" (0UL)
 	);

arch/x86/lib/checksum_32.S

@@ -28,6 +28,7 @@
 #include <linux/linkage.h>
 #include <asm/dwarf2.h>
 #include <asm/errno.h>
+#include <asm/asm.h>
 				
 /*
  * computes a partial checksum, e.g. for TCP/UDP fragments
@@ -282,15 +283,11 @@ unsigned int csum_partial_copy_generic (const char *src, char *dst,
 
 #define SRC(y...)			\
 	9999: y;			\
-	.section __ex_table, "a";	\
+	_ASM_EXTABLE(9999b, 6001f)
-	.long 9999b, 6001f	;	\
-	.previous
 
 #define DST(y...)			\
 	9999: y;			\
-	.section __ex_table, "a";	\
+	_ASM_EXTABLE(9999b, 6002f)
-	.long 9999b, 6002f	;	\
-	.previous
 
 #ifndef CONFIG_X86_USE_PPRO_CHECKSUM
 

arch/x86/lib/copy_user_64.S

@@ -16,6 +16,7 @@
 #include <asm/thread_info.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative-asm.h>
+#include <asm/asm.h>
 
 /*
  * By placing feature2 after feature1 in altinstructions section, we logically
@@ -63,11 +64,8 @@
 	jmp copy_user_handle_tail
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(100b,103b)
-	.align 8
+	_ASM_EXTABLE(101b,103b)
-	.quad 100b,103b
-	.quad 101b,103b
-	.previous
 #endif
 	.endm
 
@@ -191,29 +189,26 @@ ENTRY(copy_user_generic_unrolled)
 60:	jmp copy_user_handle_tail /* ecx is zerorest also */
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,30b)
-	.align 8
+	_ASM_EXTABLE(2b,30b)
-	.quad 1b,30b
+	_ASM_EXTABLE(3b,30b)
-	.quad 2b,30b
+	_ASM_EXTABLE(4b,30b)
-	.quad 3b,30b
+	_ASM_EXTABLE(5b,30b)
-	.quad 4b,30b
+	_ASM_EXTABLE(6b,30b)
-	.quad 5b,30b
+	_ASM_EXTABLE(7b,30b)
-	.quad 6b,30b
+	_ASM_EXTABLE(8b,30b)
-	.quad 7b,30b
+	_ASM_EXTABLE(9b,30b)
-	.quad 8b,30b
+	_ASM_EXTABLE(10b,30b)
-	.quad 9b,30b
+	_ASM_EXTABLE(11b,30b)
-	.quad 10b,30b
+	_ASM_EXTABLE(12b,30b)
-	.quad 11b,30b
+	_ASM_EXTABLE(13b,30b)
-	.quad 12b,30b
+	_ASM_EXTABLE(14b,30b)
-	.quad 13b,30b
+	_ASM_EXTABLE(15b,30b)
-	.quad 14b,30b
+	_ASM_EXTABLE(16b,30b)
-	.quad 15b,30b
+	_ASM_EXTABLE(18b,40b)
-	.quad 16b,30b
+	_ASM_EXTABLE(19b,40b)
-	.quad 18b,40b
+	_ASM_EXTABLE(21b,50b)
-	.quad 19b,40b
+	_ASM_EXTABLE(22b,50b)
-	.quad 21b,50b
-	.quad 22b,50b
-	.previous
 	CFI_ENDPROC
 ENDPROC(copy_user_generic_unrolled)
 
@@ -259,11 +254,8 @@ ENTRY(copy_user_generic_string)
 	jmp copy_user_handle_tail
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,11b)
-	.align 8
+	_ASM_EXTABLE(3b,12b)
-	.quad 1b,11b
-	.quad 3b,12b
-	.previous
 	CFI_ENDPROC
 ENDPROC(copy_user_generic_string)
 
@@ -294,9 +286,6 @@ ENTRY(copy_user_enhanced_fast_string)
 	jmp copy_user_handle_tail
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,12b)
-	.align 8
-	.quad 1b,12b
-	.previous
 	CFI_ENDPROC
 ENDPROC(copy_user_enhanced_fast_string)

arch/x86/lib/copy_user_nocache_64.S

@@ -14,6 +14,7 @@
 #include <asm/current.h>
 #include <asm/asm-offsets.h>
 #include <asm/thread_info.h>
+#include <asm/asm.h>
 
 	.macro ALIGN_DESTINATION
 #ifdef FIX_ALIGNMENT
@@ -36,11 +37,8 @@
 	jmp copy_user_handle_tail
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(100b,103b)
-	.align 8
+	_ASM_EXTABLE(101b,103b)
-	.quad 100b,103b
-	.quad 101b,103b
-	.previous
 #endif
 	.endm
 
@@ -111,27 +109,25 @@ ENTRY(__copy_user_nocache)
 	jmp copy_user_handle_tail
 	.previous
 
-	.section __ex_table,"a"
+	_ASM_EXTABLE(1b,30b)
-	.quad 1b,30b
+	_ASM_EXTABLE(2b,30b)
-	.quad 2b,30b
+	_ASM_EXTABLE(3b,30b)
-	.quad 3b,30b
+	_ASM_EXTABLE(4b,30b)
-	.quad 4b,30b
+	_ASM_EXTABLE(5b,30b)
-	.quad 5b,30b
+	_ASM_EXTABLE(6b,30b)
-	.quad 6b,30b
+	_ASM_EXTABLE(7b,30b)
-	.quad 7b,30b
+	_ASM_EXTABLE(8b,30b)
-	.quad 8b,30b
+	_ASM_EXTABLE(9b,30b)
-	.quad 9b,30b
+	_ASM_EXTABLE(10b,30b)
-	.quad 10b,30b
+	_ASM_EXTABLE(11b,30b)
-	.quad 11b,30b
+	_ASM_EXTABLE(12b,30b)
-	.quad 12b,30b
+	_ASM_EXTABLE(13b,30b)
-	.quad 13b,30b
+	_ASM_EXTABLE(14b,30b)
-	.quad 14b,30b
+	_ASM_EXTABLE(15b,30b)
-	.quad 15b,30b
+	_ASM_EXTABLE(16b,30b)
-	.quad 16b,30b
+	_ASM_EXTABLE(18b,40b)
-	.quad 18b,40b
+	_ASM_EXTABLE(19b,40b)
-	.quad 19b,40b
+	_ASM_EXTABLE(21b,50b)
-	.quad 21b,50b
+	_ASM_EXTABLE(22b,50b)
-	.quad 22b,50b
-	.previous
 	CFI_ENDPROC
 ENDPROC(__copy_user_nocache)