grsecurity-3.1-4.8.10-20161123221320161123

13 files changed, 5143 insertions, 161 deletions

diff --git a/4.8.9/0000_README b/4.8.10/0000_README
index 1062a0b..c32fdff 100644
--- a/4.8.9/0000_README
+++ b/4.8.10/0000_README
@@ -6,7 +6,11 @@ Patch:	1008_linux-4.8.9.patch
 From:	http://www.kernel.org
 Desc:	Linux 4.8.9
 
-Patch:	4420_grsecurity-3.1-4.8.9-201611192033.patch
+Patch:	1009_linux-4.8.10.patch
+From:	http://www.kernel.org
+Desc:	Linux 4.8.10
+
+Patch:	4420_grsecurity-3.1-4.8.10-201611232213.patch
 From:	http://www.grsecurity.net
 Desc:	hardened-sources base patch from upstream grsecurity
 
diff --git a/4.8.9/1008_linux-4.8.9.patch b/4.8.10/1008_linux-4.8.9.patch
index 2f909eb..2f909eb 100644
--- a/4.8.9/1008_linux-4.8.9.patch
+++ b/4.8.10/1008_linux-4.8.9.patch
diff --git a/4.8.10/1009_linux-4.8.10.patch b/4.8.10/1009_linux-4.8.10.patch
new file mode 100644
index 0000000..1e751e5
--- /dev/null
+++ b/4.8.10/1009_linux-4.8.10.patch
@@ -0,0 +1,4759 @@
+diff --git a/Makefile b/Makefile
+index c1519ab..7cf2b49 100644
+--- a/Makefile
++++ b/Makefile
+@@ -1,6 +1,6 @@
+ VERSION = 4
+ PATCHLEVEL = 8
+-SUBLEVEL = 9
++SUBLEVEL = 10
+ EXTRAVERSION =
+ NAME = Psychotic Stoned Sheep
+ 
+diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
+index 37a315d..a6847fc 100644
+--- a/arch/sparc/include/asm/uaccess_64.h
++++ b/arch/sparc/include/asm/uaccess_64.h
+@@ -98,7 +98,6 @@ struct exception_table_entry {
+         unsigned int insn, fixup;
+ };
+ 
+-void __ret_efault(void);
+ void __retl_efault(void);
+ 
+ /* Uh, these should become the main single-value transfer routines..
+@@ -205,55 +204,34 @@ int __get_user_bad(void);
+ unsigned long __must_check ___copy_from_user(void *to,
+ 					     const void __user *from,
+ 					     unsigned long size);
+-unsigned long copy_from_user_fixup(void *to, const void __user *from,
+-				   unsigned long size);
+ static inline unsigned long __must_check
+ copy_from_user(void *to, const void __user *from, unsigned long size)
+ {
+-	unsigned long ret;
+-
+ 	check_object_size(to, size, false);
+ 
+-	ret = ___copy_from_user(to, from, size);
+-	if (unlikely(ret))
+-		ret = copy_from_user_fixup(to, from, size);
+-
+-	return ret;
++	return ___copy_from_user(to, from, size);
+ }
+ #define __copy_from_user copy_from_user
+ 
+ unsigned long __must_check ___copy_to_user(void __user *to,
+ 					   const void *from,
+ 					   unsigned long size);
+-unsigned long copy_to_user_fixup(void __user *to, const void *from,
+-				 unsigned long size);
+ static inline unsigned long __must_check
+ copy_to_user(void __user *to, const void *from, unsigned long size)
+ {
+-	unsigned long ret;
+-
+ 	check_object_size(from, size, true);
+ 
+-	ret = ___copy_to_user(to, from, size);
+-	if (unlikely(ret))
+-		ret = copy_to_user_fixup(to, from, size);
+-	return ret;
++	return ___copy_to_user(to, from, size);
+ }
+ #define __copy_to_user copy_to_user
+ 
+ unsigned long __must_check ___copy_in_user(void __user *to,
+ 					   const void __user *from,
+ 					   unsigned long size);
+-unsigned long copy_in_user_fixup(void __user *to, void __user *from,
+-				 unsigned long size);
+ static inline unsigned long __must_check
+ copy_in_user(void __user *to, void __user *from, unsigned long size)
+ {
+-	unsigned long ret = ___copy_in_user(to, from, size);
+-
+-	if (unlikely(ret))
+-		ret = copy_in_user_fixup(to, from, size);
+-	return ret;
++	return ___copy_in_user(to, from, size);
+ }
+ #define __copy_in_user copy_in_user
+ 
+diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S
+index a076b42..5f1f3ae 100644
+--- a/arch/sparc/kernel/head_64.S
++++ b/arch/sparc/kernel/head_64.S
+@@ -922,47 +922,11 @@ prom_tba:	.xword	0
+ tlb_type:	.word	0	/* Must NOT end up in BSS */
+ 	.section	".fixup",#alloc,#execinstr
+ 
+-	.globl	__ret_efault, __retl_efault, __ret_one, __retl_one
+-ENTRY(__ret_efault)
+-	ret
+-	 restore %g0, -EFAULT, %o0
+-ENDPROC(__ret_efault)
+-
+ ENTRY(__retl_efault)
+ 	retl
+ 	 mov	-EFAULT, %o0
+ ENDPROC(__retl_efault)
+ 
+-ENTRY(__retl_one)
+-	retl
+-	 mov	1, %o0
+-ENDPROC(__retl_one)
+-
+-ENTRY(__retl_one_fp)
+-	VISExitHalf
+-	retl
+-	 mov	1, %o0
+-ENDPROC(__retl_one_fp)
+-
+-ENTRY(__ret_one_asi)
+-	wr	%g0, ASI_AIUS, %asi
+-	ret
+-	 restore %g0, 1, %o0
+-ENDPROC(__ret_one_asi)
+-
+-ENTRY(__retl_one_asi)
+-	wr	%g0, ASI_AIUS, %asi
+-	retl
+-	 mov	1, %o0
+-ENDPROC(__retl_one_asi)
+-
+-ENTRY(__retl_one_asi_fp)
+-	wr	%g0, ASI_AIUS, %asi
+-	VISExitHalf
+-	retl
+-	 mov	1, %o0
+-ENDPROC(__retl_one_asi_fp)
+-
+ ENTRY(__retl_o1)
+ 	retl
+ 	 mov	%o1, %o0
+diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
+index 59bbeff..07933b9 100644
+--- a/arch/sparc/kernel/jump_label.c
++++ b/arch/sparc/kernel/jump_label.c
+@@ -13,19 +13,30 @@
+ void arch_jump_label_transform(struct jump_entry *entry,
+ 			       enum jump_label_type type)
+ {
+-	u32 val;
+ 	u32 *insn = (u32 *) (unsigned long) entry->code;
++	u32 val;
+ 
+ 	if (type == JUMP_LABEL_JMP) {
+ 		s32 off = (s32)entry->target - (s32)entry->code;
++		bool use_v9_branch = false;
++
++		BUG_ON(off & 3);
+ 
+ #ifdef CONFIG_SPARC64
+-		/* ba,pt %xcc, . + (off << 2) */
+-		val = 0x10680000 | ((u32) off >> 2);
+-#else
+-		/* ba . + (off << 2) */
+-		val = 0x10800000 | ((u32) off >> 2);
++		if (off <= 0xfffff && off >= -0x100000)
++			use_v9_branch = true;
+ #endif
++		if (use_v9_branch) {
++			/* WDISP19 - target is . + immed << 2 */
++			/* ba,pt %xcc, . + off */
++			val = 0x10680000 | (((u32) off >> 2) & 0x7ffff);
++		} else {
++			/* WDISP22 - target is . + immed << 2 */
++			BUG_ON(off > 0x7fffff);
++			BUG_ON(off < -0x800000);
++			/* ba . + off */
++			val = 0x10800000 | (((u32) off >> 2) & 0x3fffff);
++		}
+ 	} else {
+ 		val = 0x01000000;
+ 	}
+diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c
+index 9e034f2..20ffb05 100644
+--- a/arch/sparc/kernel/sparc_ksyms_64.c
++++ b/arch/sparc/kernel/sparc_ksyms_64.c
+@@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user);
+ EXPORT_SYMBOL_GPL(real_hard_smp_processor_id);
+ 
+ /* from head_64.S */
+-EXPORT_SYMBOL(__ret_efault);
+ EXPORT_SYMBOL(tlb_type);
+ EXPORT_SYMBOL(sun4v_chip_type);
+ EXPORT_SYMBOL(prom_root_node);
+diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S
+index b7d0bd6..69a439f 100644
+--- a/arch/sparc/lib/GENcopy_from_user.S
++++ b/arch/sparc/lib/GENcopy_from_user.S
+@@ -3,11 +3,11 @@
+  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S
+index 780550e..9947427 100644
+--- a/arch/sparc/lib/GENcopy_to_user.S
++++ b/arch/sparc/lib/GENcopy_to_user.S
+@@ -3,11 +3,11 @@
+  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S
+index 89358ee..059ea24 100644
+--- a/arch/sparc/lib/GENmemcpy.S
++++ b/arch/sparc/lib/GENmemcpy.S
+@@ -4,21 +4,18 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #define GLOBAL_SPARE	%g7
+ #else
+ #define GLOBAL_SPARE	%g5
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
+-#endif
+-
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ 
+ #ifndef LOAD
+@@ -45,6 +42,29 @@
+ 	.register	%g3,#scratch
+ 
+ 	.text
++
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++ENTRY(GEN_retl_o4_1)
++	add	%o4, %o2, %o4
++	retl
++	 add	%o4, 1, %o0
++ENDPROC(GEN_retl_o4_1)
++ENTRY(GEN_retl_g1_8)
++	add	%g1, %o2, %g1
++	retl
++	 add	%g1, 8, %o0
++ENDPROC(GEN_retl_g1_8)
++ENTRY(GEN_retl_o2_4)
++	retl
++	 add	%o2, 4, %o0
++ENDPROC(GEN_retl_o2_4)
++ENTRY(GEN_retl_o2_1)
++	retl
++	 add	%o2, 1, %o0
++ENDPROC(GEN_retl_o2_1)
++#endif
++
+ 	.align		64
+ 
+ 	.globl	FUNC_NAME
+@@ -73,8 +93,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	sub		%g0, %o4, %o4
+ 	sub		%o2, %o4, %o2
+ 1:	subcc		%o4, 1, %o4
+-	EX_LD(LOAD(ldub, %o1, %g1))
+-	EX_ST(STORE(stb, %g1, %o0))
++	EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1)
++	EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1)
+ 	add		%o1, 1, %o1
+ 	bne,pt		%XCC, 1b
+ 	add		%o0, 1, %o0
+@@ -82,8 +102,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	andn		%o2, 0x7, %g1
+ 	sub		%o2, %g1, %o2
+ 1:	subcc		%g1, 0x8, %g1
+-	EX_LD(LOAD(ldx, %o1, %g2))
+-	EX_ST(STORE(stx, %g2, %o0))
++	EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8)
++	EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8)
+ 	add		%o1, 0x8, %o1
+ 	bne,pt		%XCC, 1b
+ 	 add		%o0, 0x8, %o0
+@@ -100,8 +120,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 1:
+ 	subcc		%o2, 4, %o2
+-	EX_LD(LOAD(lduw, %o1, %g1))
+-	EX_ST(STORE(stw, %g1, %o1 + %o3))
++	EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4)
++	EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 4, %o1
+ 
+@@ -111,8 +131,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	.align		32
+ 90:
+ 	subcc		%o2, 1, %o2
+-	EX_LD(LOAD(ldub, %o1, %g1))
+-	EX_ST(STORE(stb, %g1, %o1 + %o3))
++	EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1)
++	EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1)
+ 	bgu,pt		%XCC, 90b
+ 	 add		%o1, 1, %o1
+ 	retl
+diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
+index 3269b02..4f2384a 100644
+--- a/arch/sparc/lib/Makefile
++++ b/arch/sparc/lib/Makefile
+@@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) +=  NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o
+ lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
+ lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
+ 
+-lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o
++lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
+ lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
+ 
+ obj-$(CONFIG_SPARC64) += iomap.o
+diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S
+index d5242b8..b79a699 100644
+--- a/arch/sparc/lib/NG2copy_from_user.S
++++ b/arch/sparc/lib/NG2copy_from_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_LD_FP(x)		\
++#define EX_LD_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S
+index 4e962d9..dcec55f 100644
+--- a/arch/sparc/lib/NG2copy_to_user.S
++++ b/arch/sparc/lib/NG2copy_to_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_ST_FP(x)		\
++#define EX_ST_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
+index d5f585d..c629dbd 100644
+--- a/arch/sparc/lib/NG2memcpy.S
++++ b/arch/sparc/lib/NG2memcpy.S
+@@ -4,6 +4,7 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #include <asm/visasm.h>
+ #include <asm/asi.h>
+ #define GLOBAL_SPARE	%g7
+@@ -32,21 +33,17 @@
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ #ifndef EX_LD_FP
+-#define EX_LD_FP(x)	x
++#define EX_LD_FP(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ #ifndef EX_ST_FP
+-#define EX_ST_FP(x)	x
+-#endif
+-
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
++#define EX_ST_FP(x,y)	x
+ #endif
+ 
+ #ifndef LOAD
+@@ -140,45 +137,110 @@
+ 	fsrc2		%x6, %f12; \
+ 	fsrc2		%x7, %f14;
+ #define FREG_LOAD_1(base, x0) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0))
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1)
+ #define FREG_LOAD_2(base, x0, x1) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1);
+ #define FREG_LOAD_3(base, x0, x1, x2) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x10, %x2));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1);
+ #define FREG_LOAD_4(base, x0, x1, x2, x3) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x18, %x3));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1);
+ #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x20, %x4));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1);
+ #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x28, %x5));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1);
+ #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \
+-	EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \
+-	EX_LD_FP(LOAD(ldd, base + 0x30, %x6));
++	EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \
++	EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1);
+ 
+ 	.register	%g2,#scratch
+ 	.register	%g3,#scratch
+ 
+ 	.text
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++__restore_fp:
++	VISExitHalf
++__restore_asi:
++	retl
++	 wr	%g0, ASI_AIUS, %asi
++ENTRY(NG2_retl_o2)
++	ba,pt	%xcc, __restore_asi
++	 mov	%o2, %o0
++ENDPROC(NG2_retl_o2)
++ENTRY(NG2_retl_o2_plus_1)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, 1, %o0
++ENDPROC(NG2_retl_o2_plus_1)
++ENTRY(NG2_retl_o2_plus_4)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, 4, %o0
++ENDPROC(NG2_retl_o2_plus_4)
++ENTRY(NG2_retl_o2_plus_8)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, 8, %o0
++ENDPROC(NG2_retl_o2_plus_8)
++ENTRY(NG2_retl_o2_plus_o4_plus_1)
++	add	%o4, 1, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG2_retl_o2_plus_o4_plus_1)
++ENTRY(NG2_retl_o2_plus_o4_plus_8)
++	add	%o4, 8, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG2_retl_o2_plus_o4_plus_8)
++ENTRY(NG2_retl_o2_plus_o4_plus_16)
++	add	%o4, 16, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG2_retl_o2_plus_o4_plus_16)
++ENTRY(NG2_retl_o2_plus_g1_fp)
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %g1, %o0
++ENDPROC(NG2_retl_o2_plus_g1_fp)
++ENTRY(NG2_retl_o2_plus_g1_plus_64_fp)
++	add	%g1, 64, %g1
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %g1, %o0
++ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp)
++ENTRY(NG2_retl_o2_plus_g1_plus_1)
++	add	%g1, 1, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %g1, %o0
++ENDPROC(NG2_retl_o2_plus_g1_plus_1)
++ENTRY(NG2_retl_o2_and_7_plus_o4)
++	and	%o2, 7, %o2
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG2_retl_o2_and_7_plus_o4)
++ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8)
++	and	%o2, 7, %o2
++	add	%o4, 8, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8)
++#endif
++
+ 	.align		64
+ 
+ 	.globl	FUNC_NAME
+@@ -230,8 +292,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	sub		%g0, %o4, %o4	! bytes to align dst
+ 	sub		%o2, %o4, %o2
+ 1:	subcc		%o4, 1, %o4
+-	EX_LD(LOAD(ldub, %o1, %g1))
+-	EX_ST(STORE(stb, %g1, %o0))
++	EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1)
++	EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1)
+ 	add		%o1, 1, %o1
+ 	bne,pt		%XCC, 1b
+ 	add		%o0, 1, %o0
+@@ -281,11 +343,11 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	 nop
+ 	/* fall through for 0 < low bits < 8 */
+ 110:	sub		%o4, 64, %g2
+-	EX_LD_FP(LOAD_BLK(%g2, %f0))
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++	EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1)
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -296,10 +358,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 120:	sub		%o4, 56, %g2
+ 	FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -310,10 +372,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 130:	sub		%o4, 48, %g2
+ 	FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_6(f20, f22, f24, f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -324,10 +386,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 140:	sub		%o4, 40, %g2
+ 	FREG_LOAD_5(%g2, f0, f2, f4, f6, f8)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_5(f22, f24, f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -338,10 +400,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 150:	sub		%o4, 32, %g2
+ 	FREG_LOAD_4(%g2, f0, f2, f4, f6)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_4(f24, f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -352,10 +414,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 160:	sub		%o4, 24, %g2
+ 	FREG_LOAD_3(%g2, f0, f2, f4)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_3(f26, f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -366,10 +428,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 170:	sub		%o4, 16, %g2
+ 	FREG_LOAD_2(%g2, f0, f2)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_2(f28, f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -380,10 +442,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 180:	sub		%o4, 8, %g2
+ 	FREG_LOAD_1(%g2, f0)
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
+-	EX_LD_FP(LOAD_BLK(%o4, %f16))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
++	EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1)
+ 	FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30)
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	FREG_MOVE_1(f30)
+ 	subcc		%g1, 64, %g1
+ 	add		%o4, 64, %o4
+@@ -393,10 +455,10 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	 nop
+ 
+ 190:
+-1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3))
++1:	EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1)
+ 	subcc		%g1, 64, %g1
+-	EX_LD_FP(LOAD_BLK(%o4, %f0))
+-	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3))
++	EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64)
++	EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64)
+ 	add		%o4, 64, %o4
+ 	bne,pt		%xcc, 1b
+ 	 LOAD(prefetch, %o4 + 64, #one_read)
+@@ -423,28 +485,28 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	andn		%o2, 0xf, %o4
+ 	and		%o2, 0xf, %o2
+ 1:	subcc		%o4, 0x10, %o4
+-	EX_LD(LOAD(ldx, %o1, %o5))
++	EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16)
+ 	add		%o1, 0x08, %o1
+-	EX_LD(LOAD(ldx, %o1, %g1))
++	EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16)
+ 	sub		%o1, 0x08, %o1
+-	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
++	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16)
+ 	add		%o1, 0x8, %o1
+-	EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE))
++	EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 0x8, %o1
+ 73:	andcc		%o2, 0x8, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%o2, 0x8, %o2
+-	EX_LD(LOAD(ldx, %o1, %o5))
+-	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE))
++	EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8)
++	EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8)
+ 	add		%o1, 0x8, %o1
+ 1:	andcc		%o2, 0x4, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%o2, 0x4, %o2
+-	EX_LD(LOAD(lduw, %o1, %o5))
+-	EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE))
++	EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4)
++	EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
+ 	add		%o1, 0x4, %o1
+ 1:	cmp		%o2, 0
+ 	be,pt		%XCC, 85f
+@@ -460,8 +522,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	sub		%o2, %g1, %o2
+ 
+ 1:	subcc		%g1, 1, %g1
+-	EX_LD(LOAD(ldub, %o1, %o5))
+-	EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE))
++	EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1)
++	EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1)
+ 	bgu,pt		%icc, 1b
+ 	 add		%o1, 1, %o1
+ 
+@@ -477,16 +539,16 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 8:	mov		64, GLOBAL_SPARE
+ 	andn		%o1, 0x7, %o1
+-	EX_LD(LOAD(ldx, %o1, %g2))
++	EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2)
+ 	sub		GLOBAL_SPARE, %g1, GLOBAL_SPARE
+ 	andn		%o2, 0x7, %o4
+ 	sllx		%g2, %g1, %g2
+ 1:	add		%o1, 0x8, %o1
+-	EX_LD(LOAD(ldx, %o1, %g3))
++	EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4)
+ 	subcc		%o4, 0x8, %o4
+ 	srlx		%g3, GLOBAL_SPARE, %o5
+ 	or		%o5, %g2, %o5
+-	EX_ST(STORE(stx, %o5, %o0))
++	EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8)
+ 	add		%o0, 0x8, %o0
+ 	bgu,pt		%icc, 1b
+ 	 sllx		%g3, %g1, %g2
+@@ -506,8 +568,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 1:
+ 	subcc		%o2, 4, %o2
+-	EX_LD(LOAD(lduw, %o1, %g1))
+-	EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE))
++	EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4)
++	EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 4, %o1
+ 
+@@ -517,8 +579,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	.align		32
+ 90:
+ 	subcc		%o2, 1, %o2
+-	EX_LD(LOAD(ldub, %o1, %g1))
+-	EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE))
++	EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1)
++	EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1)
+ 	bgu,pt		%XCC, 90b
+ 	 add		%o1, 1, %o1
+ 	retl
+diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S
+index 2e8ee7a..16a286c 100644
+--- a/arch/sparc/lib/NG4copy_from_user.S
++++ b/arch/sparc/lib/NG4copy_from_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x, y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_LD_FP(x)		\
++#define EX_LD_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S
+index be0bf45..6b0276f 100644
+--- a/arch/sparc/lib/NG4copy_to_user.S
++++ b/arch/sparc/lib/NG4copy_to_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 2012 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_ST_FP(x)		\
++#define EX_ST_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_asi_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S
+index 8e13ee1..75bb93b 100644
+--- a/arch/sparc/lib/NG4memcpy.S
++++ b/arch/sparc/lib/NG4memcpy.S
+@@ -4,6 +4,7 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #include <asm/visasm.h>
+ #include <asm/asi.h>
+ #define GLOBAL_SPARE	%g7
+@@ -46,22 +47,19 @@
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ #ifndef EX_LD_FP
+-#define EX_LD_FP(x)	x
++#define EX_LD_FP(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ #ifndef EX_ST_FP
+-#define EX_ST_FP(x)	x
++#define EX_ST_FP(x,y)	x
+ #endif
+ 
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
+-#endif
+ 
+ #ifndef LOAD
+ #define LOAD(type,addr,dest)	type [addr], dest
+@@ -94,6 +92,158 @@
+ 	.register	%g3,#scratch
+ 
+ 	.text
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++__restore_asi_fp:
++	VISExitHalf
++__restore_asi:
++	retl
++	 wr	%g0, ASI_AIUS, %asi
++
++ENTRY(NG4_retl_o2)
++	ba,pt	%xcc, __restore_asi
++	 mov	%o2, %o0
++ENDPROC(NG4_retl_o2)
++ENTRY(NG4_retl_o2_plus_1)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, 1, %o0
++ENDPROC(NG4_retl_o2_plus_1)
++ENTRY(NG4_retl_o2_plus_4)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, 4, %o0
++ENDPROC(NG4_retl_o2_plus_4)
++ENTRY(NG4_retl_o2_plus_o5)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5)
++ENTRY(NG4_retl_o2_plus_o5_plus_4)
++	add	%o5, 4, %o5
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5_plus_4)
++ENTRY(NG4_retl_o2_plus_o5_plus_8)
++	add	%o5, 8, %o5
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5_plus_8)
++ENTRY(NG4_retl_o2_plus_o5_plus_16)
++	add	%o5, 16, %o5
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5_plus_16)
++ENTRY(NG4_retl_o2_plus_o5_plus_24)
++	add	%o5, 24, %o5
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5_plus_24)
++ENTRY(NG4_retl_o2_plus_o5_plus_32)
++	add	%o5, 32, %o5
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o5, %o0
++ENDPROC(NG4_retl_o2_plus_o5_plus_32)
++ENTRY(NG4_retl_o2_plus_g1)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %g1, %o0
++ENDPROC(NG4_retl_o2_plus_g1)
++ENTRY(NG4_retl_o2_plus_g1_plus_1)
++	add	%g1, 1, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %g1, %o0
++ENDPROC(NG4_retl_o2_plus_g1_plus_1)
++ENTRY(NG4_retl_o2_plus_g1_plus_8)
++	add	%g1, 8, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %g1, %o0
++ENDPROC(NG4_retl_o2_plus_g1_plus_8)
++ENTRY(NG4_retl_o2_plus_o4)
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4)
++ENTRY(NG4_retl_o2_plus_o4_plus_8)
++	add	%o4, 8, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_8)
++ENTRY(NG4_retl_o2_plus_o4_plus_16)
++	add	%o4, 16, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_16)
++ENTRY(NG4_retl_o2_plus_o4_plus_24)
++	add	%o4, 24, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_24)
++ENTRY(NG4_retl_o2_plus_o4_plus_32)
++	add	%o4, 32, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_32)
++ENTRY(NG4_retl_o2_plus_o4_plus_40)
++	add	%o4, 40, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_40)
++ENTRY(NG4_retl_o2_plus_o4_plus_48)
++	add	%o4, 48, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_48)
++ENTRY(NG4_retl_o2_plus_o4_plus_56)
++	add	%o4, 56, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_56)
++ENTRY(NG4_retl_o2_plus_o4_plus_64)
++	add	%o4, 64, %o4
++	ba,pt	%xcc, __restore_asi
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_64)
++ENTRY(NG4_retl_o2_plus_o4_fp)
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_8_fp)
++	add	%o4, 8, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_16_fp)
++	add	%o4, 16, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_24_fp)
++	add	%o4, 24, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_32_fp)
++	add	%o4, 32, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_40_fp)
++	add	%o4, 40, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_48_fp)
++	add	%o4, 48, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_56_fp)
++	add	%o4, 56, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp)
++ENTRY(NG4_retl_o2_plus_o4_plus_64_fp)
++	add	%o4, 64, %o4
++	ba,pt	%xcc, __restore_asi_fp
++	 add	%o2, %o4, %o0
++ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp)
++#endif
+ 	.align		64
+ 
+ 	.globl	FUNC_NAME
+@@ -124,12 +274,13 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	brz,pt		%g1, 51f
+ 	 sub		%o2, %g1, %o2
+ 
+-1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
++
++1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
+ 	add		%o1, 1, %o1
+ 	subcc		%g1, 1, %g1
+ 	add		%o0, 1, %o0
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stb, %g2, %o0 - 0x01))
++	 EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
+ 
+ 51:	LOAD(prefetch, %o1 + 0x040, #n_reads_strong)
+ 	LOAD(prefetch, %o1 + 0x080, #n_reads_strong)
+@@ -154,43 +305,43 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	brz,pt		%g1, .Llarge_aligned
+ 	 sub		%o2, %g1, %o2
+ 
+-1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g2))
++1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
+ 	add		%o1, 8, %o1
+ 	subcc		%g1, 8, %g1
+ 	add		%o0, 8, %o0
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stx, %g2, %o0 - 0x08))
++	 EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8)
+ 
+ .Llarge_aligned:
+ 	/* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */
+ 	andn		%o2, 0x3f, %o4
+ 	sub		%o2, %o4, %o2
+ 
+-1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
++1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4)
+ 	add		%o1, 0x40, %o1
+-	EX_LD(LOAD(ldx, %o1 - 0x38, %g2))
++	EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4)
+ 	subcc		%o4, 0x40, %o4
+-	EX_LD(LOAD(ldx, %o1 - 0x30, %g3))
+-	EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE))
+-	EX_LD(LOAD(ldx, %o1 - 0x20, %o5))
+-	EX_ST(STORE_INIT(%g1, %o0))
++	EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64)
++	EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64)
+ 	add		%o0, 0x08, %o0
+-	EX_ST(STORE_INIT(%g2, %o0))
++	EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56)
+ 	add		%o0, 0x08, %o0
+-	EX_LD(LOAD(ldx, %o1 - 0x18, %g2))
+-	EX_ST(STORE_INIT(%g3, %o0))
++	EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48)
++	EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48)
+ 	add		%o0, 0x08, %o0
+-	EX_LD(LOAD(ldx, %o1 - 0x10, %g3))
+-	EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
++	EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40)
++	EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40)
+ 	add		%o0, 0x08, %o0
+-	EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE))
+-	EX_ST(STORE_INIT(%o5, %o0))
++	EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32)
++	EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32)
+ 	add		%o0, 0x08, %o0
+-	EX_ST(STORE_INIT(%g2, %o0))
++	EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24)
+ 	add		%o0, 0x08, %o0
+-	EX_ST(STORE_INIT(%g3, %o0))
++	EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16)
+ 	add		%o0, 0x08, %o0
+-	EX_ST(STORE_INIT(GLOBAL_SPARE, %o0))
++	EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8)
+ 	add		%o0, 0x08, %o0
+ 	bne,pt		%icc, 1b
+ 	 LOAD(prefetch, %o1 + 0x200, #n_reads_strong)
+@@ -216,17 +367,17 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	sub		%o2, %o4, %o2
+ 	alignaddr	%o1, %g0, %g1
+ 	add		%o1, %o4, %o1
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0))
+-1:	EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2))
++	EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4)
++1:	EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4)
+ 	subcc		%o4, 0x40, %o4
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4))
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6))
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8))
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10))
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12))
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14))
++	EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64)
++	EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64)
+ 	faligndata	%f0, %f2, %f16
+-	EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0))
++	EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64)
+ 	faligndata	%f2, %f4, %f18
+ 	add		%g1, 0x40, %g1
+ 	faligndata	%f4, %f6, %f20
+@@ -235,14 +386,14 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	faligndata	%f10, %f12, %f26
+ 	faligndata	%f12, %f14, %f28
+ 	faligndata	%f14, %f0, %f30
+-	EX_ST_FP(STORE(std, %f16, %o0 + 0x00))
+-	EX_ST_FP(STORE(std, %f18, %o0 + 0x08))
+-	EX_ST_FP(STORE(std, %f20, %o0 + 0x10))
+-	EX_ST_FP(STORE(std, %f22, %o0 + 0x18))
+-	EX_ST_FP(STORE(std, %f24, %o0 + 0x20))
+-	EX_ST_FP(STORE(std, %f26, %o0 + 0x28))
+-	EX_ST_FP(STORE(std, %f28, %o0 + 0x30))
+-	EX_ST_FP(STORE(std, %f30, %o0 + 0x38))
++	EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64)
++	EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56)
++	EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48)
++	EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40)
++	EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32)
++	EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24)
++	EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16)
++	EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8)
+ 	add		%o0, 0x40, %o0
+ 	bne,pt		%icc, 1b
+ 	 LOAD(prefetch, %g1 + 0x200, #n_reads_strong)
+@@ -270,37 +421,38 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	andncc		%o2, 0x20 - 1, %o5
+ 	be,pn		%icc, 2f
+ 	 sub		%o2, %o5, %o2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
+-	EX_LD(LOAD(ldx, %o1 + 0x08, %g2))
+-	EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE))
+-	EX_LD(LOAD(ldx, %o1 + 0x18, %o4))
++1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
++	EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5)
++	EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5)
++	EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5)
+ 	add		%o1, 0x20, %o1
+ 	subcc		%o5, 0x20, %o5
+-	EX_ST(STORE(stx, %g1, %o0 + 0x00))
+-	EX_ST(STORE(stx, %g2, %o0 + 0x08))
+-	EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10))
+-	EX_ST(STORE(stx, %o4, %o0 + 0x18))
++	EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32)
++	EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24)
++	EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24)
++	EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8)
+ 	bne,pt		%icc, 1b
+ 	 add		%o0, 0x20, %o0
+ 2:	andcc		%o2, 0x18, %o5
+ 	be,pt		%icc, 3f
+ 	 sub		%o2, %o5, %o2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1))
++
++1:	EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
+ 	add		%o1, 0x08, %o1
+ 	add		%o0, 0x08, %o0
+ 	subcc		%o5, 0x08, %o5
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stx, %g1, %o0 - 0x08))
++	 EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8)
+ 3:	brz,pt		%o2, .Lexit
+ 	 cmp		%o2, 0x04
+ 	bl,pn		%icc, .Ltiny
+ 	 nop
+-	EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
++	EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2)
+ 	add		%o1, 0x04, %o1
+ 	add		%o0, 0x04, %o0
+ 	subcc		%o2, 0x04, %o2
+ 	bne,pn		%icc, .Ltiny
+-	 EX_ST(STORE(stw, %g1, %o0 - 0x04))
++	 EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4)
+ 	ba,a,pt		%icc, .Lexit
+ .Lmedium_unaligned:
+ 	/* First get dest 8 byte aligned.  */
+@@ -309,12 +461,12 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	brz,pt		%g1, 2f
+ 	 sub		%o2, %g1, %o2
+ 
+-1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g2))
++1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1)
+ 	add		%o1, 1, %o1
+ 	subcc		%g1, 1, %g1
+ 	add		%o0, 1, %o0
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stb, %g2, %o0 - 0x01))
++	 EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1)
+ 2:
+ 	and		%o1, 0x7, %g1
+ 	brz,pn		%g1, .Lmedium_noprefetch
+@@ -322,16 +474,16 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	mov		64, %g2
+ 	sub		%g2, %g1, %g2
+ 	andn		%o1, 0x7, %o1
+-	EX_LD(LOAD(ldx, %o1 + 0x00, %o4))
++	EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2)
+ 	sllx		%o4, %g1, %o4
+ 	andn		%o2, 0x08 - 1, %o5
+ 	sub		%o2, %o5, %o2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x08, %g3))
++1:	EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5)
+ 	add		%o1, 0x08, %o1
+ 	subcc		%o5, 0x08, %o5
+ 	srlx		%g3, %g2, GLOBAL_SPARE
+ 	or		GLOBAL_SPARE, %o4, GLOBAL_SPARE
+-	EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00))
++	EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8)
+ 	add		%o0, 0x08, %o0
+ 	bne,pt		%icc, 1b
+ 	 sllx		%g3, %g1, %o4
+@@ -342,17 +494,17 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	ba,pt		%icc, .Lsmall_unaligned
+ 
+ .Ltiny:
+-	EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
++	EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
+ 	subcc		%o2, 1, %o2
+ 	be,pn		%icc, .Lexit
+-	 EX_ST(STORE(stb, %g1, %o0 + 0x00))
+-	EX_LD(LOAD(ldub, %o1 + 0x01, %g1))
++	 EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1)
++	EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2)
+ 	subcc		%o2, 1, %o2
+ 	be,pn		%icc, .Lexit
+-	 EX_ST(STORE(stb, %g1, %o0 + 0x01))
+-	EX_LD(LOAD(ldub, %o1 + 0x02, %g1))
++	 EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1)
++	EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2)
+ 	ba,pt		%icc, .Lexit
+-	 EX_ST(STORE(stb, %g1, %o0 + 0x02))
++	 EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2)
+ 
+ .Lsmall:
+ 	andcc		%g2, 0x3, %g0
+@@ -360,22 +512,22 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	 andn		%o2, 0x4 - 1, %o5
+ 	sub		%o2, %o5, %o2
+ 1:
+-	EX_LD(LOAD(lduw, %o1 + 0x00, %g1))
++	EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5)
+ 	add		%o1, 0x04, %o1
+ 	subcc		%o5, 0x04, %o5
+ 	add		%o0, 0x04, %o0
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stw, %g1, %o0 - 0x04))
++	 EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4)
+ 	brz,pt		%o2, .Lexit
+ 	 nop
+ 	ba,a,pt		%icc, .Ltiny
+ 
+ .Lsmall_unaligned:
+-1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g1))
++1:	EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2)
+ 	add		%o1, 1, %o1
+ 	add		%o0, 1, %o0
+ 	subcc		%o2, 1, %o2
+ 	bne,pt		%icc, 1b
+-	 EX_ST(STORE(stb, %g1, %o0 - 0x01))
++	 EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1)
+ 	ba,a,pt		%icc, .Lexit
+ 	.size		FUNC_NAME, .-FUNC_NAME
+diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S
+index 5d1e4d1..9cd42fc 100644
+--- a/arch/sparc/lib/NGcopy_from_user.S
++++ b/arch/sparc/lib/NGcopy_from_user.S
+@@ -3,11 +3,11 @@
+  * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __ret_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S
+index ff630dc..5c358af 100644
+--- a/arch/sparc/lib/NGcopy_to_user.S
++++ b/arch/sparc/lib/NGcopy_to_user.S
+@@ -3,11 +3,11 @@
+  * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __ret_one_asi;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S
+index 96a14ca..d88c4ed 100644
+--- a/arch/sparc/lib/NGmemcpy.S
++++ b/arch/sparc/lib/NGmemcpy.S
+@@ -4,6 +4,7 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #include <asm/asi.h>
+ #include <asm/thread_info.h>
+ #define GLOBAL_SPARE	%g7
+@@ -27,15 +28,11 @@
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
+-#endif
+-
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ 
+ #ifndef LOAD
+@@ -79,6 +76,92 @@
+ 	.register	%g3,#scratch
+ 
+ 	.text
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++__restore_asi:
++	ret
++	wr	%g0, ASI_AIUS, %asi
++	 restore
++ENTRY(NG_ret_i2_plus_i4_plus_1)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %i5, %i0
++ENDPROC(NG_ret_i2_plus_i4_plus_1)
++ENTRY(NG_ret_i2_plus_g1)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1)
++ENTRY(NG_ret_i2_plus_g1_minus_8)
++	sub	%g1, 8, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_8)
++ENTRY(NG_ret_i2_plus_g1_minus_16)
++	sub	%g1, 16, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_16)
++ENTRY(NG_ret_i2_plus_g1_minus_24)
++	sub	%g1, 24, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_24)
++ENTRY(NG_ret_i2_plus_g1_minus_32)
++	sub	%g1, 32, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_32)
++ENTRY(NG_ret_i2_plus_g1_minus_40)
++	sub	%g1, 40, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_40)
++ENTRY(NG_ret_i2_plus_g1_minus_48)
++	sub	%g1, 48, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_48)
++ENTRY(NG_ret_i2_plus_g1_minus_56)
++	sub	%g1, 56, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_minus_56)
++ENTRY(NG_ret_i2_plus_i4)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %i4, %i0
++ENDPROC(NG_ret_i2_plus_i4)
++ENTRY(NG_ret_i2_plus_i4_minus_8)
++	sub	%i4, 8, %i4
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %i4, %i0
++ENDPROC(NG_ret_i2_plus_i4_minus_8)
++ENTRY(NG_ret_i2_plus_8)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, 8, %i0
++ENDPROC(NG_ret_i2_plus_8)
++ENTRY(NG_ret_i2_plus_4)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, 4, %i0
++ENDPROC(NG_ret_i2_plus_4)
++ENTRY(NG_ret_i2_plus_1)
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, 1, %i0
++ENDPROC(NG_ret_i2_plus_1)
++ENTRY(NG_ret_i2_plus_g1_plus_1)
++	add	%g1, 1, %g1
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %g1, %i0
++ENDPROC(NG_ret_i2_plus_g1_plus_1)
++ENTRY(NG_ret_i2)
++	ba,pt	%xcc, __restore_asi
++	 mov	%i2, %i0
++ENDPROC(NG_ret_i2)
++ENTRY(NG_ret_i2_and_7_plus_i4)
++	and	%i2, 7, %i2
++	ba,pt	%xcc, __restore_asi
++	 add	%i2, %i4, %i0
++ENDPROC(NG_ret_i2_and_7_plus_i4)
++#endif
++
+ 	.align		64
+ 
+ 	.globl	FUNC_NAME
+@@ -126,8 +209,8 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	sub		%g0, %i4, %i4	! bytes to align dst
+ 	sub		%i2, %i4, %i2
+ 1:	subcc		%i4, 1, %i4
+-	EX_LD(LOAD(ldub, %i1, %g1))
+-	EX_ST(STORE(stb, %g1, %o0))
++	EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1)
++	EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1)
+ 	add		%i1, 1, %i1
+ 	bne,pt		%XCC, 1b
+ 	add		%o0, 1, %o0
+@@ -160,7 +243,7 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	and		%i4, 0x7, GLOBAL_SPARE
+ 	sll		GLOBAL_SPARE, 3, GLOBAL_SPARE
+ 	mov		64, %i5
+-	EX_LD(LOAD_TWIN(%i1, %g2, %g3))
++	EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1)
+ 	sub		%i5, GLOBAL_SPARE, %i5
+ 	mov		16, %o4
+ 	mov		32, %o5
+@@ -178,31 +261,31 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	srlx		WORD3, PRE_SHIFT, TMP; \
+ 	or		WORD2, TMP, WORD2;
+ 
+-8:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
++8:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
+ 	MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
+ 	LOAD(prefetch, %i1 + %i3, #one_read)
+ 
+-	EX_ST(STORE_INIT(%g2, %o0 + 0x00))
+-	EX_ST(STORE_INIT(%g3, %o0 + 0x08))
++	EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1)
++	EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
++	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
+ 	MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x10))
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x18))
++	EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
++	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
+ 	MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%g2, %o0 + 0x20))
+-	EX_ST(STORE_INIT(%g3, %o0 + 0x28))
++	EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
++	EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
++	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
+ 	add		%i1, 64, %i1
+ 	MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x30))
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x38))
++	EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
++	EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
+ 
+ 	subcc		%g1, 64, %g1
+ 	bne,pt		%XCC, 8b
+@@ -211,31 +294,31 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	ba,pt		%XCC, 60f
+ 	 add		%i1, %i4, %i1
+ 
+-9:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3))
++9:	EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1)
+ 	MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
+ 	LOAD(prefetch, %i1 + %i3, #one_read)
+ 
+-	EX_ST(STORE_INIT(%g3, %o0 + 0x00))
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x08))
++	EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1)
++	EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3))
++	EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16)
+ 	MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x10))
+-	EX_ST(STORE_INIT(%g2, %o0 + 0x18))
++	EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
++	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
+ 	MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%g3, %o0 + 0x20))
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x28))
++	EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
++	EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
+ 
+-	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3))
++	EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48)
+ 	add		%i1, 64, %i1
+ 	MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1)
+ 
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x30))
+-	EX_ST(STORE_INIT(%g2, %o0 + 0x38))
++	EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
++	EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
+ 
+ 	subcc		%g1, 64, %g1
+ 	bne,pt		%XCC, 9b
+@@ -249,25 +332,25 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	 * one twin load ahead, then add 8 back into source when
+ 	 * we finish the loop.
+ 	 */
+-	EX_LD(LOAD_TWIN(%i1, %o4, %o5))
++	EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1)
+ 	mov	16, %o7
+ 	mov	32, %g2
+ 	mov	48, %g3
+ 	mov	64, %o1
+-1:	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
++1:	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
+ 	LOAD(prefetch, %i1 + %o1, #one_read)
+-	EX_ST(STORE_INIT(%o5, %o0 + 0x00))	! initializes cache line
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x08))
+-	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x10))
+-	EX_ST(STORE_INIT(%o4, %o0 + 0x18))
+-	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
+-	EX_ST(STORE_INIT(%o5, %o0 + 0x20))
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x28))
+-	EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5))
++	EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1)	! initializes cache line
++	EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
++	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
++	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
++	EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
++	EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
++	EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48)
+ 	add		%i1, 64, %i1
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x30))
+-	EX_ST(STORE_INIT(%o4, %o0 + 0x38))
++	EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
++	EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
+ 	subcc		%g1, 64, %g1
+ 	bne,pt		%XCC, 1b
+ 	 add		%o0, 64, %o0
+@@ -282,20 +365,20 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	mov	32, %g2
+ 	mov	48, %g3
+ 	mov	64, %o1
+-1:	EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5))
+-	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3))
++1:	EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1)
++	EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1)
+ 	LOAD(prefetch, %i1 + %o1, #one_read)
+-	EX_ST(STORE_INIT(%o4, %o0 + 0x00))	! initializes cache line
+-	EX_ST(STORE_INIT(%o5, %o0 + 0x08))
+-	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5))
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x10))
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x18))
+-	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3))
++	EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1)	! initializes cache line
++	EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8)
++	EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16)
++	EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24)
++	EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32)
+ 	add	%i1, 64, %i1
+-	EX_ST(STORE_INIT(%o4, %o0 + 0x20))
+-	EX_ST(STORE_INIT(%o5, %o0 + 0x28))
+-	EX_ST(STORE_INIT(%o2, %o0 + 0x30))
+-	EX_ST(STORE_INIT(%o3, %o0 + 0x38))
++	EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32)
++	EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40)
++	EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48)
++	EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56)
+ 	subcc	%g1, 64, %g1
+ 	bne,pt	%XCC, 1b
+ 	 add	%o0, 64, %o0
+@@ -321,28 +404,28 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	andn		%i2, 0xf, %i4
+ 	and		%i2, 0xf, %i2
+ 1:	subcc		%i4, 0x10, %i4
+-	EX_LD(LOAD(ldx, %i1, %o4))
++	EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4)
+ 	add		%i1, 0x08, %i1
+-	EX_LD(LOAD(ldx, %i1, %g1))
++	EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4)
+ 	sub		%i1, 0x08, %i1
+-	EX_ST(STORE(stx, %o4, %i1 + %i3))
++	EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4)
+ 	add		%i1, 0x8, %i1
+-	EX_ST(STORE(stx, %g1, %i1 + %i3))
++	EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%i1, 0x8, %i1
+ 73:	andcc		%i2, 0x8, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%i2, 0x8, %i2
+-	EX_LD(LOAD(ldx, %i1, %o4))
+-	EX_ST(STORE(stx, %o4, %i1 + %i3))
++	EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8)
++	EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8)
+ 	add		%i1, 0x8, %i1
+ 1:	andcc		%i2, 0x4, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%i2, 0x4, %i2
+-	EX_LD(LOAD(lduw, %i1, %i5))
+-	EX_ST(STORE(stw, %i5, %i1 + %i3))
++	EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4)
++	EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4)
+ 	add		%i1, 0x4, %i1
+ 1:	cmp		%i2, 0
+ 	be,pt		%XCC, 85f
+@@ -358,8 +441,8 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	sub		%i2, %g1, %i2
+ 
+ 1:	subcc		%g1, 1, %g1
+-	EX_LD(LOAD(ldub, %i1, %i5))
+-	EX_ST(STORE(stb, %i5, %i1 + %i3))
++	EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1)
++	EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1)
+ 	bgu,pt		%icc, 1b
+ 	 add		%i1, 1, %i1
+ 
+@@ -375,16 +458,16 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 
+ 8:	mov		64, %i3
+ 	andn		%i1, 0x7, %i1
+-	EX_LD(LOAD(ldx, %i1, %g2))
++	EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2)
+ 	sub		%i3, %g1, %i3
+ 	andn		%i2, 0x7, %i4
+ 	sllx		%g2, %g1, %g2
+ 1:	add		%i1, 0x8, %i1
+-	EX_LD(LOAD(ldx, %i1, %g3))
++	EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4)
+ 	subcc		%i4, 0x8, %i4
+ 	srlx		%g3, %i3, %i5
+ 	or		%i5, %g2, %i5
+-	EX_ST(STORE(stx, %i5, %o0))
++	EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4)
+ 	add		%o0, 0x8, %o0
+ 	bgu,pt		%icc, 1b
+ 	 sllx		%g3, %g1, %g2
+@@ -404,8 +487,8 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 
+ 1:
+ 	subcc		%i2, 4, %i2
+-	EX_LD(LOAD(lduw, %i1, %g1))
+-	EX_ST(STORE(stw, %g1, %i1 + %i3))
++	EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4)
++	EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%i1, 4, %i1
+ 
+@@ -415,8 +498,8 @@ FUNC_NAME:	/* %i0=dst, %i1=src, %i2=len */
+ 	.align		32
+ 90:
+ 	subcc		%i2, 1, %i2
+-	EX_LD(LOAD(ldub, %i1, %g1))
+-	EX_ST(STORE(stb, %g1, %i1 + %i3))
++	EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1)
++	EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1)
+ 	bgu,pt		%XCC, 90b
+ 	 add		%i1, 1, %i1
+ 	ret
+diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S
+index ecc5692..bb6ff73 100644
+--- a/arch/sparc/lib/U1copy_from_user.S
++++ b/arch/sparc/lib/U1copy_from_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_LD_FP(x)		\
++#define EX_LD_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_fp;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S
+index 9eea392..ed92ce73 100644
+--- a/arch/sparc/lib/U1copy_to_user.S
++++ b/arch/sparc/lib/U1copy_to_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_ST_FP(x)		\
++#define EX_ST_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_fp;\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
+index 3e6209e..f30d2ab 100644
+--- a/arch/sparc/lib/U1memcpy.S
++++ b/arch/sparc/lib/U1memcpy.S
+@@ -5,6 +5,7 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #include <asm/visasm.h>
+ #include <asm/asi.h>
+ #define GLOBAL_SPARE	g7
+@@ -23,21 +24,17 @@
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ #ifndef EX_LD_FP
+-#define EX_LD_FP(x)	x
++#define EX_LD_FP(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ #ifndef EX_ST_FP
+-#define EX_ST_FP(x)	x
+-#endif
+-
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
++#define EX_ST_FP(x,y)	x
+ #endif
+ 
+ #ifndef LOAD
+@@ -78,53 +75,169 @@
+ 	faligndata		%f7, %f8, %f60;			\
+ 	faligndata		%f8, %f9, %f62;
+ 
+-#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt)	\
+-	EX_LD_FP(LOAD_BLK(%src, %fdest));				\
+-	EX_ST_FP(STORE_BLK(%fsrc, %dest));				\
+-	add			%src, 0x40, %src;		\
+-	subcc			%len, 0x40, %len;		\
+-	be,pn			%xcc, jmptgt;			\
+-	 add			%dest, 0x40, %dest;		\
+-
+-#define LOOP_CHUNK1(src, dest, len, branch_dest)		\
+-	MAIN_LOOP_CHUNK(src, dest, f0,  f48, len, branch_dest)
+-#define LOOP_CHUNK2(src, dest, len, branch_dest)		\
+-	MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
+-#define LOOP_CHUNK3(src, dest, len, branch_dest)		\
+-	MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
++#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt)			\
++	EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp);			\
++	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp);			\
++	add			%src, 0x40, %src;			\
++	subcc			%GLOBAL_SPARE, 0x40, %GLOBAL_SPARE;	\
++	be,pn			%xcc, jmptgt;				\
++	 add			%dest, 0x40, %dest;			\
++
++#define LOOP_CHUNK1(src, dest, branch_dest)		\
++	MAIN_LOOP_CHUNK(src, dest, f0,  f48, branch_dest)
++#define LOOP_CHUNK2(src, dest, branch_dest)		\
++	MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest)
++#define LOOP_CHUNK3(src, dest, branch_dest)		\
++	MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest)
+ 
+ #define DO_SYNC			membar	#Sync;
+ #define STORE_SYNC(dest, fsrc)				\
+-	EX_ST_FP(STORE_BLK(%fsrc, %dest));			\
++	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp);	\
+ 	add			%dest, 0x40, %dest;	\
+ 	DO_SYNC
+ 
+ #define STORE_JUMP(dest, fsrc, target)			\
+-	EX_ST_FP(STORE_BLK(%fsrc, %dest));			\
++	EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp);	\
+ 	add			%dest, 0x40, %dest;	\
+ 	ba,pt			%xcc, target;		\
+ 	 nop;
+ 
+-#define FINISH_VISCHUNK(dest, f0, f1, left)	\
+-	subcc			%left, 8, %left;\
+-	bl,pn			%xcc, 95f;	\
+-	 faligndata		%f0, %f1, %f48;	\
+-	EX_ST_FP(STORE(std, %f48, %dest));		\
++#define FINISH_VISCHUNK(dest, f0, f1)			\
++	subcc			%g3, 8, %g3;		\
++	bl,pn			%xcc, 95f;		\
++	 faligndata		%f0, %f1, %f48;		\
++	EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp);	\
+ 	add			%dest, 8, %dest;
+ 
+-#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
+-	subcc			%left, 8, %left;	\
+-	bl,pn			%xcc, 95f;		\
++#define UNEVEN_VISCHUNK_LAST(dest, f0, f1)	\
++	subcc			%g3, 8, %g3;	\
++	bl,pn			%xcc, 95f;	\
+ 	 fsrc2			%f0, %f1;
+ 
+-#define UNEVEN_VISCHUNK(dest, f0, f1, left)		\
+-	UNEVEN_VISCHUNK_LAST(dest, f0, f1, left)	\
++#define UNEVEN_VISCHUNK(dest, f0, f1)		\
++	UNEVEN_VISCHUNK_LAST(dest, f0, f1)	\
+ 	ba,a,pt			%xcc, 93f;
+ 
+ 	.register	%g2,#scratch
+ 	.register	%g3,#scratch
+ 
+ 	.text
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++ENTRY(U1_g1_1_fp)
++	VISExitHalf
++	add		%g1, 1, %g1
++	add		%g1, %g2, %g1
++	retl
++	 add		%g1, %o2, %o0
++ENDPROC(U1_g1_1_fp)
++ENTRY(U1_g2_0_fp)
++	VISExitHalf
++	retl
++	 add		%g2, %o2, %o0
++ENDPROC(U1_g2_0_fp)
++ENTRY(U1_g2_8_fp)
++	VISExitHalf
++	add		%g2, 8, %g2
++	retl
++	 add		%g2, %o2, %o0
++ENDPROC(U1_g2_8_fp)
++ENTRY(U1_gs_0_fp)
++	VISExitHalf
++	add		%GLOBAL_SPARE, %g3, %o0
++	retl
++	 add		%o0, %o2, %o0
++ENDPROC(U1_gs_0_fp)
++ENTRY(U1_gs_80_fp)
++	VISExitHalf
++	add		%GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
++	add		%GLOBAL_SPARE, %g3, %o0
++	retl
++	 add		%o0, %o2, %o0
++ENDPROC(U1_gs_80_fp)
++ENTRY(U1_gs_40_fp)
++	VISExitHalf
++	add		%GLOBAL_SPARE, 0x40, %GLOBAL_SPARE
++	add		%GLOBAL_SPARE, %g3, %o0
++	retl
++	 add		%o0, %o2, %o0
++ENDPROC(U1_gs_40_fp)
++ENTRY(U1_g3_0_fp)
++	VISExitHalf
++	retl
++	 add		%g3, %o2, %o0
++ENDPROC(U1_g3_0_fp)
++ENTRY(U1_g3_8_fp)
++	VISExitHalf
++	add		%g3, 8, %g3
++	retl
++	 add		%g3, %o2, %o0
++ENDPROC(U1_g3_8_fp)
++ENTRY(U1_o2_0_fp)
++	VISExitHalf
++	retl
++	 mov		%o2, %o0
++ENDPROC(U1_o2_0_fp)
++ENTRY(U1_o2_1_fp)
++	VISExitHalf
++	retl
++	 add		%o2, 1, %o0
++ENDPROC(U1_o2_1_fp)
++ENTRY(U1_gs_0)
++	VISExitHalf
++	retl
++	 add		%GLOBAL_SPARE, %o2, %o0
++ENDPROC(U1_gs_0)
++ENTRY(U1_gs_8)
++	VISExitHalf
++	add		%GLOBAL_SPARE, %o2, %GLOBAL_SPARE
++	retl
++	 add		%GLOBAL_SPARE, 0x8, %o0
++ENDPROC(U1_gs_8)
++ENTRY(U1_gs_10)
++	VISExitHalf
++	add		%GLOBAL_SPARE, %o2, %GLOBAL_SPARE
++	retl
++	 add		%GLOBAL_SPARE, 0x10, %o0
++ENDPROC(U1_gs_10)
++ENTRY(U1_o2_0)
++	retl
++	 mov		%o2, %o0
++ENDPROC(U1_o2_0)
++ENTRY(U1_o2_8)
++	retl
++	 add		%o2, 8, %o0
++ENDPROC(U1_o2_8)
++ENTRY(U1_o2_4)
++	retl
++	 add		%o2, 4, %o0
++ENDPROC(U1_o2_4)
++ENTRY(U1_o2_1)
++	retl
++	 add		%o2, 1, %o0
++ENDPROC(U1_o2_1)
++ENTRY(U1_g1_0)
++	retl
++	 add		%g1, %o2, %o0
++ENDPROC(U1_g1_0)
++ENTRY(U1_g1_1)
++	add		%g1, 1, %g1
++	retl
++	 add		%g1, %o2, %o0
++ENDPROC(U1_g1_1)
++ENTRY(U1_gs_0_o2_adj)
++	and		%o2, 7, %o2
++	retl
++	 add		%GLOBAL_SPARE, %o2, %o0
++ENDPROC(U1_gs_0_o2_adj)
++ENTRY(U1_gs_8_o2_adj)
++	and		%o2, 7, %o2
++	add		%GLOBAL_SPARE, 8, %GLOBAL_SPARE
++	retl
++	 add		%GLOBAL_SPARE, %o2, %o0
++ENDPROC(U1_gs_8_o2_adj)
++#endif
++
+ 	.align		64
+ 
+ 	.globl		FUNC_NAME
+@@ -166,8 +279,8 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	 and		%g2, 0x38, %g2
+ 
+ 1:	subcc		%g1, 0x1, %g1
+-	EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
+-	EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
++	EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp)
++	EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 0x1, %o1
+ 
+@@ -178,20 +291,20 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	be,pt		%icc, 3f
+ 	 alignaddr	%o1, %g0, %o1
+ 
+-	EX_LD_FP(LOAD(ldd, %o1, %f4))
+-1:	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
++	EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp)
++1:	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f4, %f6, %f0
+-	EX_ST_FP(STORE(std, %f0, %o0))
++	EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
+ 	be,pn		%icc, 3f
+ 	 add		%o0, 0x8, %o0
+ 
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f6, %f4, %f0
+-	EX_ST_FP(STORE(std, %f0, %o0))
++	EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp)
+ 	bne,pt		%icc, 1b
+ 	 add		%o0, 0x8, %o0
+ 
+@@ -214,13 +327,13 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	add		%g1, %GLOBAL_SPARE, %g1
+ 	subcc		%o2, %g3, %o2
+ 
+-	EX_LD_FP(LOAD_BLK(%o1, %f0))
++	EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp)
+ 	add		%o1, 0x40, %o1
+ 	add		%g1, %g3, %g1
+-	EX_LD_FP(LOAD_BLK(%o1, %f16))
++	EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp)
+ 	add		%o1, 0x40, %o1
+ 	sub		%GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
+-	EX_LD_FP(LOAD_BLK(%o1, %f32))
++	EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp)
+ 	add		%o1, 0x40, %o1
+ 
+ 	/* There are 8 instances of the unrolled loop,
+@@ -240,11 +353,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 
+ 	.align		64
+ 1:	FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f0, %f2, %f48
+ 1:	FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
+@@ -261,11 +374,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 56f)
+ 
+ 1:	FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f2, %f4, %f48
+ 1:	FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
+@@ -282,11 +395,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 57f)
+ 
+ 1:	FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f4, %f6, %f48
+ 1:	FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
+@@ -303,11 +416,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 58f)
+ 
+ 1:	FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) 
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f6, %f8, %f48
+ 1:	FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
+@@ -324,11 +437,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 59f)
+ 
+ 1:	FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f8, %f10, %f48
+ 1:	FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
+@@ -345,11 +458,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 60f)
+ 
+ 1:	FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f10, %f12, %f48
+ 1:	FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
+@@ -366,11 +479,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 61f)
+ 
+ 1:	FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f12, %f14, %f48
+ 1:	FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
+@@ -387,11 +500,11 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	STORE_JUMP(o0, f48, 62f)
+ 
+ 1:	FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
+-	LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
++	LOOP_CHUNK1(o1, o0, 1f)
+ 	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+-	LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
++	LOOP_CHUNK2(o1, o0, 2f)
+ 	FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
+-	LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
++	LOOP_CHUNK3(o1, o0, 3f)
+ 	ba,pt		%xcc, 1b+4
+ 	 faligndata	%f14, %f16, %f48
+ 1:	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+@@ -407,53 +520,53 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
+ 	STORE_JUMP(o0, f48, 63f)
+ 
+-40:	FINISH_VISCHUNK(o0, f0,  f2,  g3)
+-41:	FINISH_VISCHUNK(o0, f2,  f4,  g3)
+-42:	FINISH_VISCHUNK(o0, f4,  f6,  g3)
+-43:	FINISH_VISCHUNK(o0, f6,  f8,  g3)
+-44:	FINISH_VISCHUNK(o0, f8,  f10, g3)
+-45:	FINISH_VISCHUNK(o0, f10, f12, g3)
+-46:	FINISH_VISCHUNK(o0, f12, f14, g3)
+-47:	UNEVEN_VISCHUNK(o0, f14, f0,  g3)
+-48:	FINISH_VISCHUNK(o0, f16, f18, g3)
+-49:	FINISH_VISCHUNK(o0, f18, f20, g3)
+-50:	FINISH_VISCHUNK(o0, f20, f22, g3)
+-51:	FINISH_VISCHUNK(o0, f22, f24, g3)
+-52:	FINISH_VISCHUNK(o0, f24, f26, g3)
+-53:	FINISH_VISCHUNK(o0, f26, f28, g3)
+-54:	FINISH_VISCHUNK(o0, f28, f30, g3)
+-55:	UNEVEN_VISCHUNK(o0, f30, f0,  g3)
+-56:	FINISH_VISCHUNK(o0, f32, f34, g3)
+-57:	FINISH_VISCHUNK(o0, f34, f36, g3)
+-58:	FINISH_VISCHUNK(o0, f36, f38, g3)
+-59:	FINISH_VISCHUNK(o0, f38, f40, g3)
+-60:	FINISH_VISCHUNK(o0, f40, f42, g3)
+-61:	FINISH_VISCHUNK(o0, f42, f44, g3)
+-62:	FINISH_VISCHUNK(o0, f44, f46, g3)
+-63:	UNEVEN_VISCHUNK_LAST(o0, f46, f0,  g3)
+-
+-93:	EX_LD_FP(LOAD(ldd, %o1, %f2))
++40:	FINISH_VISCHUNK(o0, f0,  f2)
++41:	FINISH_VISCHUNK(o0, f2,  f4)
++42:	FINISH_VISCHUNK(o0, f4,  f6)
++43:	FINISH_VISCHUNK(o0, f6,  f8)
++44:	FINISH_VISCHUNK(o0, f8,  f10)
++45:	FINISH_VISCHUNK(o0, f10, f12)
++46:	FINISH_VISCHUNK(o0, f12, f14)
++47:	UNEVEN_VISCHUNK(o0, f14, f0)
++48:	FINISH_VISCHUNK(o0, f16, f18)
++49:	FINISH_VISCHUNK(o0, f18, f20)
++50:	FINISH_VISCHUNK(o0, f20, f22)
++51:	FINISH_VISCHUNK(o0, f22, f24)
++52:	FINISH_VISCHUNK(o0, f24, f26)
++53:	FINISH_VISCHUNK(o0, f26, f28)
++54:	FINISH_VISCHUNK(o0, f28, f30)
++55:	UNEVEN_VISCHUNK(o0, f30, f0)
++56:	FINISH_VISCHUNK(o0, f32, f34)
++57:	FINISH_VISCHUNK(o0, f34, f36)
++58:	FINISH_VISCHUNK(o0, f36, f38)
++59:	FINISH_VISCHUNK(o0, f38, f40)
++60:	FINISH_VISCHUNK(o0, f40, f42)
++61:	FINISH_VISCHUNK(o0, f42, f44)
++62:	FINISH_VISCHUNK(o0, f44, f46)
++63:	UNEVEN_VISCHUNK_LAST(o0, f46, f0)
++
++93:	EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp)
+ 	add		%o1, 8, %o1
+ 	subcc		%g3, 8, %g3
+ 	faligndata	%f0, %f2, %f8
+-	EX_ST_FP(STORE(std, %f8, %o0))
++	EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
+ 	bl,pn		%xcc, 95f
+ 	 add		%o0, 8, %o0
+-	EX_LD_FP(LOAD(ldd, %o1, %f0))
++	EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp)
+ 	add		%o1, 8, %o1
+ 	subcc		%g3, 8, %g3
+ 	faligndata	%f2, %f0, %f8
+-	EX_ST_FP(STORE(std, %f8, %o0))
++	EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp)
+ 	bge,pt		%xcc, 93b
+ 	 add		%o0, 8, %o0
+ 
+ 95:	brz,pt		%o2, 2f
+ 	 mov		%g1, %o1
+ 
+-1:	EX_LD_FP(LOAD(ldub, %o1, %o3))
++1:	EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp)
+ 	add		%o1, 1, %o1
+ 	subcc		%o2, 1, %o2
+-	EX_ST_FP(STORE(stb, %o3, %o0))
++	EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp)
+ 	bne,pt		%xcc, 1b
+ 	 add		%o0, 1, %o0
+ 
+@@ -469,27 +582,27 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 
+ 72:	andn		%o2, 0xf, %GLOBAL_SPARE
+ 	and		%o2, 0xf, %o2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+-	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
++1:	EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0)
++	EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0)
+ 	subcc		%GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
+-	EX_ST(STORE(stx, %o5, %o1 + %o3))
++	EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10)
+ 	add		%o1, 0x8, %o1
+-	EX_ST(STORE(stx, %g1, %o1 + %o3))
++	EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 0x8, %o1
+ 73:	andcc		%o2, 0x8, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+-	EX_LD(LOAD(ldx, %o1, %o5))
++	EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0)
+ 	sub		%o2, 0x8, %o2
+-	EX_ST(STORE(stx, %o5, %o1 + %o3))
++	EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8)
+ 	add		%o1, 0x8, %o1
+ 1:	andcc		%o2, 0x4, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+-	EX_LD(LOAD(lduw, %o1, %o5))
++	EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0)
+ 	sub		%o2, 0x4, %o2
+-	EX_ST(STORE(stw, %o5, %o1 + %o3))
++	EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4)
+ 	add		%o1, 0x4, %o1
+ 1:	cmp		%o2, 0
+ 	be,pt		%XCC, 85f
+@@ -503,9 +616,9 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	 sub		%g0, %g1, %g1
+ 	sub		%o2, %g1, %o2
+ 
+-1:	EX_LD(LOAD(ldub, %o1, %o5))
++1:	EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0)
+ 	subcc		%g1, 1, %g1
+-	EX_ST(STORE(stb, %o5, %o1 + %o3))
++	EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1)
+ 	bgu,pt		%icc, 1b
+ 	 add		%o1, 1, %o1
+ 
+@@ -521,16 +634,16 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 
+ 8:	mov		64, %o3
+ 	andn		%o1, 0x7, %o1
+-	EX_LD(LOAD(ldx, %o1, %g2))
++	EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0)
+ 	sub		%o3, %g1, %o3
+ 	andn		%o2, 0x7, %GLOBAL_SPARE
+ 	sllx		%g2, %g1, %g2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
++1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj)
+ 	subcc		%GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
+ 	add		%o1, 0x8, %o1
+ 	srlx		%g3, %o3, %o5
+ 	or		%o5, %g2, %o5
+-	EX_ST(STORE(stx, %o5, %o0))
++	EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj)
+ 	add		%o0, 0x8, %o0
+ 	bgu,pt		%icc, 1b
+ 	 sllx		%g3, %g1, %g2
+@@ -548,9 +661,9 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	bne,pn		%XCC, 90f
+ 	 sub		%o0, %o1, %o3
+ 
+-1:	EX_LD(LOAD(lduw, %o1, %g1))
++1:	EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0)
+ 	subcc		%o2, 4, %o2
+-	EX_ST(STORE(stw, %g1, %o1 + %o3))
++	EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 4, %o1
+ 
+@@ -558,9 +671,9 @@ FUNC_NAME:		/* %o0=dst, %o1=src, %o2=len */
+ 	 mov		EX_RETVAL(%o4), %o0
+ 
+ 	.align		32
+-90:	EX_LD(LOAD(ldub, %o1, %g1))
++90:	EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0)
+ 	subcc		%o2, 1, %o2
+-	EX_ST(STORE(stb, %g1, %o1 + %o3))
++	EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1)
+ 	bgu,pt		%XCC, 90b
+ 	 add		%o1, 1, %o1
+ 	retl
+diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S
+index 88ad73d..db73010 100644
+--- a/arch/sparc/lib/U3copy_from_user.S
++++ b/arch/sparc/lib/U3copy_from_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+  */
+ 
+-#define EX_LD(x)		\
++#define EX_LD(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_LD_FP(x)		\
++#define EX_LD_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S
+index 845139d..c4ee858 100644
+--- a/arch/sparc/lib/U3copy_to_user.S
++++ b/arch/sparc/lib/U3copy_to_user.S
+@@ -3,19 +3,19 @@
+  * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com)
+  */
+ 
+-#define EX_ST(x)		\
++#define EX_ST(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, y;		\
+ 	.text;			\
+ 	.align 4;
+ 
+-#define EX_ST_FP(x)		\
++#define EX_ST_FP(x,y)		\
+ 98:	x;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one_fp;\
++	.word 98b, y##_fp;	\
+ 	.text;			\
+ 	.align 4;
+ 
+diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S
+index 491ee69..54f9870 100644
+--- a/arch/sparc/lib/U3memcpy.S
++++ b/arch/sparc/lib/U3memcpy.S
+@@ -4,6 +4,7 @@
+  */
+ 
+ #ifdef __KERNEL__
++#include <linux/linkage.h>
+ #include <asm/visasm.h>
+ #include <asm/asi.h>
+ #define GLOBAL_SPARE	%g7
+@@ -22,21 +23,17 @@
+ #endif
+ 
+ #ifndef EX_LD
+-#define EX_LD(x)	x
++#define EX_LD(x,y)	x
+ #endif
+ #ifndef EX_LD_FP
+-#define EX_LD_FP(x)	x
++#define EX_LD_FP(x,y)	x
+ #endif
+ 
+ #ifndef EX_ST
+-#define EX_ST(x)	x
++#define EX_ST(x,y)	x
+ #endif
+ #ifndef EX_ST_FP
+-#define EX_ST_FP(x)	x
+-#endif
+-
+-#ifndef EX_RETVAL
+-#define EX_RETVAL(x)	x
++#define EX_ST_FP(x,y)	x
+ #endif
+ 
+ #ifndef LOAD
+@@ -77,6 +74,87 @@
+ 	 */
+ 
+ 	.text
++#ifndef EX_RETVAL
++#define EX_RETVAL(x)	x
++__restore_fp:
++	VISExitHalf
++	retl
++	 nop
++ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
++	add	%g1, 1, %g1
++	add	%g2, %g1, %g2
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %g2, %o0
++ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp)
++ENTRY(U3_retl_o2_plus_g2_fp)
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %g2, %o0
++ENDPROC(U3_retl_o2_plus_g2_fp)
++ENTRY(U3_retl_o2_plus_g2_plus_8_fp)
++	add	%g2, 8, %g2
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %g2, %o0
++ENDPROC(U3_retl_o2_plus_g2_plus_8_fp)
++ENTRY(U3_retl_o2)
++	retl
++	 mov	%o2, %o0
++ENDPROC(U3_retl_o2)
++ENTRY(U3_retl_o2_plus_1)
++	retl
++	 add	%o2, 1, %o0
++ENDPROC(U3_retl_o2_plus_1)
++ENTRY(U3_retl_o2_plus_4)
++	retl
++	 add	%o2, 4, %o0
++ENDPROC(U3_retl_o2_plus_4)
++ENTRY(U3_retl_o2_plus_8)
++	retl
++	 add	%o2, 8, %o0
++ENDPROC(U3_retl_o2_plus_8)
++ENTRY(U3_retl_o2_plus_g1_plus_1)
++	add	%g1, 1, %g1
++	retl
++	 add	%o2, %g1, %o0
++ENDPROC(U3_retl_o2_plus_g1_plus_1)
++ENTRY(U3_retl_o2_fp)
++	ba,pt	%xcc, __restore_fp
++	 mov	%o2, %o0
++ENDPROC(U3_retl_o2_fp)
++ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
++	sll	%o3, 6, %o3
++	add	%o3, 0x80, %o3
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %o3, %o0
++ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp)
++ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
++	sll	%o3, 6, %o3
++	add	%o3, 0x40, %o3
++	ba,pt	%xcc, __restore_fp
++	 add	%o2, %o3, %o0
++ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp)
++ENTRY(U3_retl_o2_plus_GS_plus_0x10)
++	add	GLOBAL_SPARE, 0x10, GLOBAL_SPARE
++	retl
++	 add	%o2, GLOBAL_SPARE, %o0
++ENDPROC(U3_retl_o2_plus_GS_plus_0x10)
++ENTRY(U3_retl_o2_plus_GS_plus_0x08)
++	add	GLOBAL_SPARE, 0x08, GLOBAL_SPARE
++	retl
++	 add	%o2, GLOBAL_SPARE, %o0
++ENDPROC(U3_retl_o2_plus_GS_plus_0x08)
++ENTRY(U3_retl_o2_and_7_plus_GS)
++	and	%o2, 7, %o2
++	retl
++	 add	%o2, GLOBAL_SPARE, %o2
++ENDPROC(U3_retl_o2_and_7_plus_GS)
++ENTRY(U3_retl_o2_and_7_plus_GS_plus_8)
++	add	GLOBAL_SPARE, 8, GLOBAL_SPARE
++	and	%o2, 7, %o2
++	retl
++	 add	%o2, GLOBAL_SPARE, %o2
++ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8)
++#endif
++
+ 	.align		64
+ 
+ 	/* The cheetah's flexible spine, oversized liver, enlarged heart,
+@@ -126,8 +204,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	 and		%g2, 0x38, %g2
+ 
+ 1:	subcc		%g1, 0x1, %g1
+-	EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
+-	EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE))
++	EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1)
++	EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 0x1, %o1
+ 
+@@ -138,20 +216,20 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	be,pt		%icc, 3f
+ 	 alignaddr	%o1, %g0, %o1
+ 
+-	EX_LD_FP(LOAD(ldd, %o1, %f4))
+-1:	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
++	EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2)
++1:	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f4, %f6, %f0
+-	EX_ST_FP(STORE(std, %f0, %o0))
++	EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8)
+ 	be,pn		%icc, 3f
+ 	 add		%o0, 0x8, %o0
+ 
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f6, %f4, %f2
+-	EX_ST_FP(STORE(std, %f2, %o0))
++	EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8)
+ 	bne,pt		%icc, 1b
+ 	 add		%o0, 0x8, %o0
+ 
+@@ -161,25 +239,25 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	LOAD(prefetch, %o1 + 0x080, #one_read)
+ 	LOAD(prefetch, %o1 + 0x0c0, #one_read)
+ 	LOAD(prefetch, %o1 + 0x100, #one_read)
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2)
+ 	LOAD(prefetch, %o1 + 0x140, #one_read)
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2)
+ 	LOAD(prefetch, %o1 + 0x180, #one_read)
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2)
+ 	LOAD(prefetch, %o1 + 0x1c0, #one_read)
+ 	faligndata	%f0, %f2, %f16
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2)
+ 	faligndata	%f2, %f4, %f18
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2)
+ 	faligndata	%f4, %f6, %f20
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2)
+ 	faligndata	%f6, %f8, %f22
+ 
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2)
+ 	faligndata	%f8, %f10, %f24
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2)
+ 	faligndata	%f10, %f12, %f26
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2)
+ 
+ 	subcc		GLOBAL_SPARE, 0x80, GLOBAL_SPARE
+ 	add		%o1, 0x40, %o1
+@@ -190,26 +268,26 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 	.align		64
+ 1:
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 	faligndata	%f12, %f14, %f28
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 	faligndata	%f14, %f0, %f30
+-	EX_ST_FP(STORE_BLK(%f16, %o0))
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
++	EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
++	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f0, %f2, %f16
+ 	add		%o0, 0x40, %o0
+ 
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f2, %f4, %f18
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f4, %f6, %f20
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	subcc		%o3, 0x01, %o3
+ 	faligndata	%f6, %f8, %f22
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 
+ 	faligndata	%f8, %f10, %f24
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 	LOAD(prefetch, %o1 + 0x1c0, #one_read)
+ 	faligndata	%f10, %f12, %f26
+ 	bg,pt		%XCC, 1b
+@@ -217,29 +295,29 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 	/* Finally we copy the last full 64-byte block. */
+ 2:
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 	faligndata	%f12, %f14, %f28
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80)
+ 	faligndata	%f14, %f0, %f30
+-	EX_ST_FP(STORE_BLK(%f16, %o0))
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6))
++	EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80)
++	EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f0, %f2, %f16
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f2, %f4, %f18
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f4, %f6, %f20
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f6, %f8, %f22
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	faligndata	%f8, %f10, %f24
+ 	cmp		%g1, 0
+ 	be,pt		%XCC, 1f
+ 	 add		%o0, 0x40, %o0
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 1:	faligndata	%f10, %f12, %f26
+ 	faligndata	%f12, %f14, %f28
+ 	faligndata	%f14, %f0, %f30
+-	EX_ST_FP(STORE_BLK(%f16, %o0))
++	EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40)
+ 	add		%o0, 0x40, %o0
+ 	add		%o1, 0x40, %o1
+ 	membar		#Sync
+@@ -259,20 +337,20 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 	sub		%o2, %g2, %o2
+ 	be,a,pt		%XCC, 1f
+-	 EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0))
++	 EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2)
+ 
+-1:	EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2))
++1:	EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f0, %f2, %f8
+-	EX_ST_FP(STORE(std, %f8, %o0))
++	EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
+ 	be,pn		%XCC, 2f
+ 	 add		%o0, 0x8, %o0
+-	EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0))
++	EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2)
+ 	add		%o1, 0x8, %o1
+ 	subcc		%g2, 0x8, %g2
+ 	faligndata	%f2, %f0, %f8
+-	EX_ST_FP(STORE(std, %f8, %o0))
++	EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8)
+ 	bne,pn		%XCC, 1b
+ 	 add		%o0, 0x8, %o0
+ 
+@@ -292,30 +370,33 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	 andcc		%o2, 0x8, %g0
+ 	be,pt		%icc, 1f
+ 	 nop
+-	EX_LD(LOAD(ldx, %o1, %o5))
+-	EX_ST(STORE(stx, %o5, %o1 + %o3))
++	EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2)
++	EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2)
+ 	add		%o1, 0x8, %o1
++	sub		%o2, 8, %o2
+ 
+ 1:	andcc		%o2, 0x4, %g0
+ 	be,pt		%icc, 1f
+ 	 nop
+-	EX_LD(LOAD(lduw, %o1, %o5))
+-	EX_ST(STORE(stw, %o5, %o1 + %o3))
++	EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2)
++	EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2)
+ 	add		%o1, 0x4, %o1
++	sub		%o2, 4, %o2
+ 
+ 1:	andcc		%o2, 0x2, %g0
+ 	be,pt		%icc, 1f
+ 	 nop
+-	EX_LD(LOAD(lduh, %o1, %o5))
+-	EX_ST(STORE(sth, %o5, %o1 + %o3))
++	EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2)
++	EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2)
+ 	add		%o1, 0x2, %o1
++	sub		%o2, 2, %o2
+ 
+ 1:	andcc		%o2, 0x1, %g0
+ 	be,pt		%icc, 85f
+ 	 nop
+-	EX_LD(LOAD(ldub, %o1, %o5))
++	EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2)
+ 	ba,pt		%xcc, 85f
+-	 EX_ST(STORE(stb, %o5, %o1 + %o3))
++	 EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2)
+ 
+ 	.align		64
+ 70: /* 16 < len <= 64 */
+@@ -326,26 +407,26 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	andn		%o2, 0xf, GLOBAL_SPARE
+ 	and		%o2, 0xf, %o2
+ 1:	subcc		GLOBAL_SPARE, 0x10, GLOBAL_SPARE
+-	EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
+-	EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
+-	EX_ST(STORE(stx, %o5, %o1 + %o3))
++	EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10)
++	EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10)
++	EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10)
+ 	add		%o1, 0x8, %o1
+-	EX_ST(STORE(stx, %g1, %o1 + %o3))
++	EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 0x8, %o1
+ 73:	andcc		%o2, 0x8, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%o2, 0x8, %o2
+-	EX_LD(LOAD(ldx, %o1, %o5))
+-	EX_ST(STORE(stx, %o5, %o1 + %o3))
++	EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8)
++	EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8)
+ 	add		%o1, 0x8, %o1
+ 1:	andcc		%o2, 0x4, %g0
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%o2, 0x4, %o2
+-	EX_LD(LOAD(lduw, %o1, %o5))
+-	EX_ST(STORE(stw, %o5, %o1 + %o3))
++	EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4)
++	EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4)
+ 	add		%o1, 0x4, %o1
+ 1:	cmp		%o2, 0
+ 	be,pt		%XCC, 85f
+@@ -361,8 +442,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	sub		%o2, %g1, %o2
+ 
+ 1:	subcc		%g1, 1, %g1
+-	EX_LD(LOAD(ldub, %o1, %o5))
+-	EX_ST(STORE(stb, %o5, %o1 + %o3))
++	EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1)
++	EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1)
+ 	bgu,pt		%icc, 1b
+ 	 add		%o1, 1, %o1
+ 
+@@ -378,16 +459,16 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 8:	mov		64, %o3
+ 	andn		%o1, 0x7, %o1
+-	EX_LD(LOAD(ldx, %o1, %g2))
++	EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2)
+ 	sub		%o3, %g1, %o3
+ 	andn		%o2, 0x7, GLOBAL_SPARE
+ 	sllx		%g2, %g1, %g2
+-1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
++1:	EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS)
+ 	subcc		GLOBAL_SPARE, 0x8, GLOBAL_SPARE
+ 	add		%o1, 0x8, %o1
+ 	srlx		%g3, %o3, %o5
+ 	or		%o5, %g2, %o5
+-	EX_ST(STORE(stx, %o5, %o0))
++	EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8)
+ 	add		%o0, 0x8, %o0
+ 	bgu,pt		%icc, 1b
+ 	 sllx		%g3, %g1, %g2
+@@ -407,8 +488,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 1:
+ 	subcc		%o2, 4, %o2
+-	EX_LD(LOAD(lduw, %o1, %g1))
+-	EX_ST(STORE(stw, %g1, %o1 + %o3))
++	EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4)
++	EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4)
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o1, 4, %o1
+ 
+@@ -418,8 +499,8 @@ FUNC_NAME:	/* %o0=dst, %o1=src, %o2=len */
+ 	.align		32
+ 90:
+ 	subcc		%o2, 1, %o2
+-	EX_LD(LOAD(ldub, %o1, %g1))
+-	EX_ST(STORE(stb, %g1, %o1 + %o3))
++	EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1)
++	EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1)
+ 	bgu,pt		%XCC, 90b
+ 	 add		%o1, 1, %o1
+ 	retl
+diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S
+index 302c0e6..4c89b48 100644
+--- a/arch/sparc/lib/copy_in_user.S
++++ b/arch/sparc/lib/copy_in_user.S
+@@ -8,18 +8,33 @@
+ 
+ #define XCC xcc
+ 
+-#define EX(x,y)			\
++#define EX(x,y,z)		\
+ 98:	x,y;			\
+ 	.section __ex_table,"a";\
+ 	.align 4;		\
+-	.word 98b, __retl_one;	\
++	.word 98b, z;		\
+ 	.text;			\
+ 	.align 4;
+ 
++#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8)
++#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4)
++#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1)
++
+ 	.register	%g2,#scratch
+ 	.register	%g3,#scratch
+ 
+ 	.text
++__retl_o4_plus_8:
++	add	%o4, %o2, %o4
++	retl
++	 add	%o4, 8, %o0
++__retl_o2_plus_4:
++	retl
++	 add	%o2, 4, %o0
++__retl_o2_plus_1:
++	retl
++	 add	%o2, 1, %o0
++
+ 	.align	32
+ 
+ 	/* Don't try to get too fancy here, just nice and
+@@ -44,8 +59,8 @@ ENTRY(___copy_in_user)	/* %o0=dst, %o1=src, %o2=len */
+ 	andn		%o2, 0x7, %o4
+ 	and		%o2, 0x7, %o2
+ 1:	subcc		%o4, 0x8, %o4
+-	EX(ldxa [%o1] %asi, %o5)
+-	EX(stxa %o5, [%o0] %asi)
++	EX_O4(ldxa [%o1] %asi, %o5)
++	EX_O4(stxa %o5, [%o0] %asi)
+ 	add		%o1, 0x8, %o1
+ 	bgu,pt		%XCC, 1b
+ 	 add		%o0, 0x8, %o0
+@@ -53,8 +68,8 @@ ENTRY(___copy_in_user)	/* %o0=dst, %o1=src, %o2=len */
+ 	be,pt		%XCC, 1f
+ 	 nop
+ 	sub		%o2, 0x4, %o2
+-	EX(lduwa [%o1] %asi, %o5)
+-	EX(stwa %o5, [%o0] %asi)
++	EX_O2_4(lduwa [%o1] %asi, %o5)
++	EX_O2_4(stwa %o5, [%o0] %asi)
+ 	add		%o1, 0x4, %o1
+ 	add		%o0, 0x4, %o0
+ 1:	cmp		%o2, 0
+@@ -70,8 +85,8 @@ ENTRY(___copy_in_user)	/* %o0=dst, %o1=src, %o2=len */
+ 
+ 82:
+ 	subcc		%o2, 4, %o2
+-	EX(lduwa [%o1] %asi, %g1)
+-	EX(stwa %g1, [%o0] %asi)
++	EX_O2_4(lduwa [%o1] %asi, %g1)
++	EX_O2_4(stwa %g1, [%o0] %asi)
+ 	add		%o1, 4, %o1
+ 	bgu,pt		%XCC, 82b
+ 	 add		%o0, 4, %o0
+@@ -82,8 +97,8 @@ ENTRY(___copy_in_user)	/* %o0=dst, %o1=src, %o2=len */
+ 	.align	32
+ 90:
+ 	subcc		%o2, 1, %o2
+-	EX(lduba [%o1] %asi, %g1)
+-	EX(stba %g1, [%o0] %asi)
++	EX_O2_1(lduba [%o1] %asi, %g1)
++	EX_O2_1(stba %g1, [%o0] %asi)
+ 	add		%o1, 1, %o1
+ 	bgu,pt		%XCC, 90b
+ 	 add		%o0, 1, %o0
+diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c
+deleted file mode 100644
+index ac96ae2..0000000
+--- a/arch/sparc/lib/user_fixup.c
++++ /dev/null
+@@ -1,71 +0,0 @@
+-/* user_fixup.c: Fix up user copy faults.
+- *
+- * Copyright (C) 2004 David S. Miller <davem@redhat.com>
+- */
+-
+-#include <linux/compiler.h>
+-#include <linux/kernel.h>
+-#include <linux/string.h>
+-#include <linux/errno.h>
+-#include <linux/module.h>
+-
+-#include <asm/uaccess.h>
+-
+-/* Calculating the exact fault address when using
+- * block loads and stores can be very complicated.
+- *
+- * Instead of trying to be clever and handling all
+- * of the cases, just fix things up simply here.
+- */
+-
+-static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset)
+-{
+-	unsigned long fault_addr = current_thread_info()->fault_address;
+-	unsigned long end = start + size;
+-
+-	if (fault_addr < start || fault_addr >= end) {
+-		*offset = 0;
+-	} else {
+-		*offset = fault_addr - start;
+-		size = end - fault_addr;
+-	}
+-	return size;
+-}
+-
+-unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size)
+-{
+-	unsigned long offset;
+-
+-	size = compute_size((unsigned long) from, size, &offset);
+-	if (likely(size))
+-		memset(to + offset, 0, size);
+-
+-	return size;
+-}
+-EXPORT_SYMBOL(copy_from_user_fixup);
+-
+-unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size)
+-{
+-	unsigned long offset;
+-
+-	return compute_size((unsigned long) to, size, &offset);
+-}
+-EXPORT_SYMBOL(copy_to_user_fixup);
+-
+-unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size)
+-{
+-	unsigned long fault_addr = current_thread_info()->fault_address;
+-	unsigned long start = (unsigned long) to;
+-	unsigned long end = start + size;
+-
+-	if (fault_addr >= start && fault_addr < end)
+-		return end - fault_addr;
+-
+-	start = (unsigned long) from;
+-	end = start + size;
+-	if (fault_addr >= start && fault_addr < end)
+-		return end - fault_addr;
+-
+-	return size;
+-}
+-EXPORT_SYMBOL(copy_in_user_fixup);
+diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
+index f2b7711..e20fbba 100644
+--- a/arch/sparc/mm/tsb.c
++++ b/arch/sparc/mm/tsb.c
+@@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr)
+ 	return (tag == (vaddr >> 22));
+ }
+ 
++static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end)
++{
++	unsigned long idx;
++
++	for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) {
++		struct tsb *ent = &swapper_tsb[idx];
++		unsigned long match = idx << 13;
++
++		match |= (ent->tag << 22);
++		if (match >= start && match < end)
++			ent->tag = (1UL << TSB_TAG_INVALID_BIT);
++	}
++}
++
+ /* TSB flushes need only occur on the processor initiating the address
+  * space modification, not on each cpu the address space has run on.
+  * Only the TLB flush needs that treatment.
+@@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
+ {
+ 	unsigned long v;
+ 
++	if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES)
++		return flush_tsb_kernel_range_scan(start, end);
++
+ 	for (v = start; v < end; v += PAGE_SIZE) {
+ 		unsigned long hash = tsb_hash(v, PAGE_SHIFT,
+ 					      KERNEL_TSB_NENTRIES);
+diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
+index b4f4733..5d2fd6c 100644
+--- a/arch/sparc/mm/ultra.S
++++ b/arch/sparc/mm/ultra.S
+@@ -30,7 +30,7 @@
+ 	.text
+ 	.align		32
+ 	.globl		__flush_tlb_mm
+-__flush_tlb_mm:		/* 18 insns */
++__flush_tlb_mm:		/* 19 insns */
+ 	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
+ 	ldxa		[%o1] ASI_DMMU, %g2
+ 	cmp		%g2, %o0
+@@ -81,7 +81,7 @@ __flush_tlb_page:	/* 22 insns */
+ 
+ 	.align		32
+ 	.globl		__flush_tlb_pending
+-__flush_tlb_pending:	/* 26 insns */
++__flush_tlb_pending:	/* 27 insns */
+ 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
+ 	rdpr		%pstate, %g7
+ 	sllx		%o1, 3, %o1
+@@ -113,12 +113,14 @@ __flush_tlb_pending:	/* 26 insns */
+ 
+ 	.align		32
+ 	.globl		__flush_tlb_kernel_range
+-__flush_tlb_kernel_range:	/* 16 insns */
++__flush_tlb_kernel_range:	/* 31 insns */
+ 	/* %o0=start, %o1=end */
+ 	cmp		%o0, %o1
+ 	be,pn		%xcc, 2f
++	 sub		%o1, %o0, %o3
++	srlx		%o3, 18, %o4
++	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
+ 	 sethi		%hi(PAGE_SIZE), %o4
+-	sub		%o1, %o0, %o3
+ 	sub		%o3, %o4, %o3
+ 	or		%o0, 0x20, %o0		! Nucleus
+ 1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
+@@ -131,6 +133,41 @@ __flush_tlb_kernel_range:	/* 16 insns */
+ 	retl
+ 	 nop
+ 	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++
++__spitfire_flush_tlb_kernel_range_slow:
++	mov		63 * 8, %o4
++1:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
++	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
++	bne,pn		%xcc, 2f
++	 mov		TLB_TAG_ACCESS, %o3
++	stxa		%g0, [%o3] ASI_IMMU
++	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
++	membar		#Sync
++2:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
++	andcc		%o3, 0x40, %g0
++	bne,pn		%xcc, 2f
++	 mov		TLB_TAG_ACCESS, %o3
++	stxa		%g0, [%o3] ASI_DMMU
++	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
++	membar		#Sync
++2:	sub		%o4, 8, %o4
++	brgez,pt	%o4, 1b
++	 nop
++	retl
++	 nop
+ 
+ __spitfire_flush_tlb_mm_slow:
+ 	rdpr		%pstate, %g1
+@@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending:	/* 27 insns */
+ 	retl
+ 	 wrpr		%g7, 0x0, %pstate
+ 
++__cheetah_flush_tlb_kernel_range:	/* 31 insns */
++	/* %o0=start, %o1=end */
++	cmp		%o0, %o1
++	be,pn		%xcc, 2f
++	 sub		%o1, %o0, %o3
++	srlx		%o3, 18, %o4
++	brnz,pn		%o4, 3f
++	 sethi		%hi(PAGE_SIZE), %o4
++	sub		%o3, %o4, %o3
++	or		%o0, 0x20, %o0		! Nucleus
++1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
++	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
++	membar		#Sync
++	brnz,pt		%o3, 1b
++	 sub		%o3, %o4, %o3
++2:	sethi		%hi(KERNBASE), %o3
++	flush		%o3
++	retl
++	 nop
++3:	mov		0x80, %o4
++	stxa		%g0, [%o4] ASI_DMMU_DEMAP
++	membar		#Sync
++	stxa		%g0, [%o4] ASI_IMMU_DEMAP
++	membar		#Sync
++	retl
++	 nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++
+ #ifdef DCACHE_ALIASING_POSSIBLE
+ __cheetah_flush_dcache_page: /* 11 insns */
+ 	sethi		%hi(PAGE_OFFSET), %g1
+@@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error:
+ 	ret
+ 	 restore
+ 
+-__hypervisor_flush_tlb_mm: /* 10 insns */
++__hypervisor_flush_tlb_mm: /* 19 insns */
+ 	mov		%o0, %o2	/* ARG2: mmu context */
+ 	mov		0, %o0		/* ARG0: CPU lists unimplemented */
+ 	mov		0, %o1		/* ARG1: CPU lists unimplemented */
+ 	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
+ 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+ 	ta		HV_FAST_TRAP
+-	brnz,pn		%o0, __hypervisor_tlb_tl0_error
++	brnz,pn		%o0, 1f
+ 	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
+ 	retl
+ 	 nop
++1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
++	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
++	 nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
+ 
+-__hypervisor_flush_tlb_page: /* 11 insns */
++__hypervisor_flush_tlb_page: /* 22 insns */
+ 	/* %o0 = context, %o1 = vaddr */
+ 	mov		%o0, %g2
+ 	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
+@@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */
+ 	srlx		%o0, PAGE_SHIFT, %o0
+ 	sllx		%o0, PAGE_SHIFT, %o0
+ 	ta		HV_MMU_UNMAP_ADDR_TRAP
+-	brnz,pn		%o0, __hypervisor_tlb_tl0_error
++	brnz,pn		%o0, 1f
+ 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+ 	retl
+ 	 nop
++1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
++	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
++	 nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
+ 
+-__hypervisor_flush_tlb_pending: /* 16 insns */
++__hypervisor_flush_tlb_pending: /* 27 insns */
+ 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
+ 	sllx		%o1, 3, %g1
+ 	mov		%o2, %g2
+@@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */
+ 	srlx		%o0, PAGE_SHIFT, %o0
+ 	sllx		%o0, PAGE_SHIFT, %o0
+ 	ta		HV_MMU_UNMAP_ADDR_TRAP
+-	brnz,pn		%o0, __hypervisor_tlb_tl0_error
++	brnz,pn		%o0, 1f
+ 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+ 	brnz,pt		%g1, 1b
+ 	 nop
+ 	retl
+ 	 nop
++1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
++	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
++	 nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
+ 
+-__hypervisor_flush_tlb_kernel_range: /* 16 insns */
++__hypervisor_flush_tlb_kernel_range: /* 31 insns */
+ 	/* %o0=start, %o1=end */
+ 	cmp		%o0, %o1
+ 	be,pn		%xcc, 2f
+-	 sethi		%hi(PAGE_SIZE), %g3
+-	mov		%o0, %g1
+-	sub		%o1, %g1, %g2
++	 sub		%o1, %o0, %g2
++	srlx		%g2, 18, %g3
++	brnz,pn		%g3, 4f
++	 mov		%o0, %g1
++	sethi		%hi(PAGE_SIZE), %g3
+ 	sub		%g2, %g3, %g2
+ 1:	add		%g1, %g2, %o0	/* ARG0: virtual address */
+ 	mov		0, %o1		/* ARG1: mmu context */
+ 	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
+ 	ta		HV_MMU_UNMAP_ADDR_TRAP
+-	brnz,pn		%o0, __hypervisor_tlb_tl0_error
++	brnz,pn		%o0, 3f
+ 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
+ 	brnz,pt		%g2, 1b
+ 	 sub		%g2, %g3, %g2
+ 2:	retl
+ 	 nop
++3:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
++	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
++	 nop
++4:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
++	mov		0, %o1		/* ARG1: CPU lists unimplemented */
++	mov		0, %o2		/* ARG2: mmu context == nucleus */
++	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
++	mov		HV_FAST_MMU_DEMAP_CTX, %o5
++	ta		HV_FAST_TRAP
++	brnz,pn		%o0, 3b
++	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
++	retl
++	 nop
+ 
+ #ifdef DCACHE_ALIASING_POSSIBLE
+ 	/* XXX Niagara and friends have an 8K cache, so no aliasing is
+@@ -394,43 +511,6 @@ tlb_patch_one:
+ 	retl
+ 	 nop
+ 
+-	.globl		cheetah_patch_cachetlbops
+-cheetah_patch_cachetlbops:
+-	save		%sp, -128, %sp
+-
+-	sethi		%hi(__flush_tlb_mm), %o0
+-	or		%o0, %lo(__flush_tlb_mm), %o0
+-	sethi		%hi(__cheetah_flush_tlb_mm), %o1
+-	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
+-	call		tlb_patch_one
+-	 mov		19, %o2
+-
+-	sethi		%hi(__flush_tlb_page), %o0
+-	or		%o0, %lo(__flush_tlb_page), %o0
+-	sethi		%hi(__cheetah_flush_tlb_page), %o1
+-	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
+-	call		tlb_patch_one
+-	 mov		22, %o2
+-
+-	sethi		%hi(__flush_tlb_pending), %o0
+-	or		%o0, %lo(__flush_tlb_pending), %o0
+-	sethi		%hi(__cheetah_flush_tlb_pending), %o1
+-	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
+-	call		tlb_patch_one
+-	 mov		27, %o2
+-
+-#ifdef DCACHE_ALIASING_POSSIBLE
+-	sethi		%hi(__flush_dcache_page), %o0
+-	or		%o0, %lo(__flush_dcache_page), %o0
+-	sethi		%hi(__cheetah_flush_dcache_page), %o1
+-	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
+-	call		tlb_patch_one
+-	 mov		11, %o2
+-#endif /* DCACHE_ALIASING_POSSIBLE */
+-
+-	ret
+-	 restore
+-
+ #ifdef CONFIG_SMP
+ 	/* These are all called by the slaves of a cross call, at
+ 	 * trap level 1, with interrupts fully disabled.
+@@ -447,7 +527,7 @@ cheetah_patch_cachetlbops:
+ 	 */
+ 	.align		32
+ 	.globl		xcall_flush_tlb_mm
+-xcall_flush_tlb_mm:	/* 21 insns */
++xcall_flush_tlb_mm:	/* 24 insns */
+ 	mov		PRIMARY_CONTEXT, %g2
+ 	ldxa		[%g2] ASI_DMMU, %g3
+ 	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
+@@ -469,9 +549,12 @@ xcall_flush_tlb_mm:	/* 21 insns */
+ 	nop
+ 	nop
+ 	nop
++	nop
++	nop
++	nop
+ 
+ 	.globl		xcall_flush_tlb_page
+-xcall_flush_tlb_page:	/* 17 insns */
++xcall_flush_tlb_page:	/* 20 insns */
+ 	/* %g5=context, %g1=vaddr */
+ 	mov		PRIMARY_CONTEXT, %g4
+ 	ldxa		[%g4] ASI_DMMU, %g2
+@@ -490,15 +573,20 @@ xcall_flush_tlb_page:	/* 17 insns */
+ 	retry
+ 	nop
+ 	nop
++	nop
++	nop
++	nop
+ 
+ 	.globl		xcall_flush_tlb_kernel_range
+-xcall_flush_tlb_kernel_range:	/* 25 insns */
++xcall_flush_tlb_kernel_range:	/* 44 insns */
+ 	sethi		%hi(PAGE_SIZE - 1), %g2
+ 	or		%g2, %lo(PAGE_SIZE - 1), %g2
+ 	andn		%g1, %g2, %g1
+ 	andn		%g7, %g2, %g7
+ 	sub		%g7, %g1, %g3
+-	add		%g2, 1, %g2
++	srlx		%g3, 18, %g2
++	brnz,pn		%g2, 2f
++	 add		%g2, 1, %g2
+ 	sub		%g3, %g2, %g3
+ 	or		%g1, 0x20, %g1		! Nucleus
+ 1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
+@@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range:	/* 25 insns */
+ 	brnz,pt		%g3, 1b
+ 	 sub		%g3, %g2, %g3
+ 	retry
+-	nop
+-	nop
++2:	mov		63 * 8, %g1
++1:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
++	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
++	bne,pn		%xcc, 2f
++	 mov		TLB_TAG_ACCESS, %g2
++	stxa		%g0, [%g2] ASI_IMMU
++	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
++	membar		#Sync
++2:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
++	andcc		%g2, 0x40, %g0
++	bne,pn		%xcc, 2f
++	 mov		TLB_TAG_ACCESS, %g2
++	stxa		%g0, [%g2] ASI_DMMU
++	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
++	membar		#Sync
++2:	sub		%g1, 8, %g1
++	brgez,pt	%g1, 1b
++	 nop
++	retry
+ 	nop
+ 	nop
+ 	nop
+@@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4:
+ 
+ 	retry
+ 
++__cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
++	sethi		%hi(PAGE_SIZE - 1), %g2
++	or		%g2, %lo(PAGE_SIZE - 1), %g2
++	andn		%g1, %g2, %g1
++	andn		%g7, %g2, %g7
++	sub		%g7, %g1, %g3
++	srlx		%g3, 18, %g2
++	brnz,pn		%g2, 2f
++	 add		%g2, 1, %g2
++	sub		%g3, %g2, %g3
++	or		%g1, 0x20, %g1		! Nucleus
++1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
++	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
++	membar		#Sync
++	brnz,pt		%g3, 1b
++	 sub		%g3, %g2, %g3
++	retry
++2:	mov		0x80, %g2
++	stxa		%g0, [%g2] ASI_DMMU_DEMAP
++	membar		#Sync
++	stxa		%g0, [%g2] ASI_IMMU_DEMAP
++	membar		#Sync
++	retry
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++	nop
++
+ #ifdef DCACHE_ALIASING_POSSIBLE
+ 	.align		32
+ 	.globl		xcall_flush_dcache_page_cheetah
+@@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error:
+ 	ba,a,pt	%xcc, rtrap
+ 
+ 	.globl		__hypervisor_xcall_flush_tlb_mm
+-__hypervisor_xcall_flush_tlb_mm: /* 21 insns */
++__hypervisor_xcall_flush_tlb_mm: /* 24 insns */
+ 	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
+ 	mov		%o0, %g2
+ 	mov		%o1, %g3
+@@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
+ 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
+ 	ta		HV_FAST_TRAP
+ 	mov		HV_FAST_MMU_DEMAP_CTX, %g6
+-	brnz,pn		%o0, __hypervisor_tlb_xcall_error
++	brnz,pn		%o0, 1f
+ 	 mov		%o0, %g5
+ 	mov		%g2, %o0
+ 	mov		%g3, %o1
+@@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
+ 	mov		%g7, %o5
+ 	membar		#Sync
+ 	retry
++1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
++	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
++	 nop
+ 
+ 	.globl		__hypervisor_xcall_flush_tlb_page
+-__hypervisor_xcall_flush_tlb_page: /* 17 insns */
++__hypervisor_xcall_flush_tlb_page: /* 20 insns */
+ 	/* %g5=ctx, %g1=vaddr */
+ 	mov		%o0, %g2
+ 	mov		%o1, %g3
+@@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */
+ 	sllx		%o0, PAGE_SHIFT, %o0
+ 	ta		HV_MMU_UNMAP_ADDR_TRAP
+ 	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
+-	brnz,a,pn	%o0, __hypervisor_tlb_xcall_error
++	brnz,a,pn	%o0, 1f
+ 	 mov		%o0, %g5
+ 	mov		%g2, %o0
+ 	mov		%g3, %o1
+ 	mov		%g4, %o2
+ 	membar		#Sync
+ 	retry
++1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
++	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
++	 nop
+ 
+ 	.globl		__hypervisor_xcall_flush_tlb_kernel_range
+-__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */
++__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
+ 	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
+ 	sethi		%hi(PAGE_SIZE - 1), %g2
+ 	or		%g2, %lo(PAGE_SIZE - 1), %g2
+ 	andn		%g1, %g2, %g1
+ 	andn		%g7, %g2, %g7
+ 	sub		%g7, %g1, %g3
++	srlx		%g3, 18, %g7
+ 	add		%g2, 1, %g2
+ 	sub		%g3, %g2, %g3
+ 	mov		%o0, %g2
+ 	mov		%o1, %g4
+-	mov		%o2, %g7
++	brnz,pn		%g7, 2f
++	 mov		%o2, %g7
+ 1:	add		%g1, %g3, %o0	/* ARG0: virtual address */
+ 	mov		0, %o1		/* ARG1: mmu context */
+ 	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
+ 	ta		HV_MMU_UNMAP_ADDR_TRAP
+ 	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
+-	brnz,pn		%o0, __hypervisor_tlb_xcall_error
++	brnz,pn		%o0, 1f
+ 	 mov		%o0, %g5
+ 	sethi		%hi(PAGE_SIZE), %o2
+ 	brnz,pt		%g3, 1b
+ 	 sub		%g3, %o2, %g3
+-	mov		%g2, %o0
++5:	mov		%g2, %o0
+ 	mov		%g4, %o1
+ 	mov		%g7, %o2
+ 	membar		#Sync
+ 	retry
++1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
++	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
++	 nop
++2:	mov		%o3, %g1
++	mov		%o5, %g3
++	mov		0, %o0		/* ARG0: CPU lists unimplemented */
++	mov		0, %o1		/* ARG1: CPU lists unimplemented */
++	mov		0, %o2		/* ARG2: mmu context == nucleus */
++	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
++	mov		HV_FAST_MMU_DEMAP_CTX, %o5
++	ta		HV_FAST_TRAP
++	mov		%g1, %o3
++	brz,pt		%o0, 5b
++	 mov		%g3, %o5
++	mov		HV_FAST_MMU_DEMAP_CTX, %g6
++	ba,pt		%xcc, 1b
++	 clr		%g5
+ 
+ 	/* These just get rescheduled to PIL vectors. */
+ 	.globl		xcall_call_function
+@@ -809,6 +985,58 @@ xcall_kgdb_capture:
+ 
+ #endif /* CONFIG_SMP */
+ 
++	.globl		cheetah_patch_cachetlbops
++cheetah_patch_cachetlbops:
++	save		%sp, -128, %sp
++
++	sethi		%hi(__flush_tlb_mm), %o0
++	or		%o0, %lo(__flush_tlb_mm), %o0
++	sethi		%hi(__cheetah_flush_tlb_mm), %o1
++	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
++	call		tlb_patch_one
++	 mov		19, %o2
++
++	sethi		%hi(__flush_tlb_page), %o0
++	or		%o0, %lo(__flush_tlb_page), %o0
++	sethi		%hi(__cheetah_flush_tlb_page), %o1
++	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
++	call		tlb_patch_one
++	 mov		22, %o2
++
++	sethi		%hi(__flush_tlb_pending), %o0
++	or		%o0, %lo(__flush_tlb_pending), %o0
++	sethi		%hi(__cheetah_flush_tlb_pending), %o1
++	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
++	call		tlb_patch_one
++	 mov		27, %o2
++
++	sethi		%hi(__flush_tlb_kernel_range), %o0
++	or		%o0, %lo(__flush_tlb_kernel_range), %o0
++	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
++	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
++	call		tlb_patch_one
++	 mov		31, %o2
++
++#ifdef DCACHE_ALIASING_POSSIBLE
++	sethi		%hi(__flush_dcache_page), %o0
++	or		%o0, %lo(__flush_dcache_page), %o0
++	sethi		%hi(__cheetah_flush_dcache_page), %o1
++	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
++	call		tlb_patch_one
++	 mov		11, %o2
++#endif /* DCACHE_ALIASING_POSSIBLE */
++
++#ifdef CONFIG_SMP
++	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
++	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
++	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
++	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
++	call		tlb_patch_one
++	 mov		44, %o2
++#endif /* CONFIG_SMP */
++
++	ret
++	 restore
+ 
+ 	.globl		hypervisor_patch_cachetlbops
+ hypervisor_patch_cachetlbops:
+@@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops:
+ 	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
+ 	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
+ 	call		tlb_patch_one
+-	 mov		10, %o2
++	 mov		19, %o2
+ 
+ 	sethi		%hi(__flush_tlb_page), %o0
+ 	or		%o0, %lo(__flush_tlb_page), %o0
+ 	sethi		%hi(__hypervisor_flush_tlb_page), %o1
+ 	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
+ 	call		tlb_patch_one
+-	 mov		11, %o2
++	 mov		22, %o2
+ 
+ 	sethi		%hi(__flush_tlb_pending), %o0
+ 	or		%o0, %lo(__flush_tlb_pending), %o0
+ 	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
+ 	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
+ 	call		tlb_patch_one
+-	 mov		16, %o2
++	 mov		27, %o2
+ 
+ 	sethi		%hi(__flush_tlb_kernel_range), %o0
+ 	or		%o0, %lo(__flush_tlb_kernel_range), %o0
+ 	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
+ 	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
+ 	call		tlb_patch_one
+-	 mov		16, %o2
++	 mov		31, %o2
+ 
+ #ifdef DCACHE_ALIASING_POSSIBLE
+ 	sethi		%hi(__flush_dcache_page), %o0
+@@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops:
+ 	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
+ 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
+ 	call		tlb_patch_one
+-	 mov		21, %o2
++	 mov		24, %o2
+ 
+ 	sethi		%hi(xcall_flush_tlb_page), %o0
+ 	or		%o0, %lo(xcall_flush_tlb_page), %o0
+ 	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
+ 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
+ 	call		tlb_patch_one
+-	 mov		17, %o2
++	 mov		20, %o2
+ 
+ 	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
+ 	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
+ 	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
+ 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
+ 	call		tlb_patch_one
+-	 mov		25, %o2
++	 mov		44, %o2
+ #endif /* CONFIG_SMP */
+ 
+ 	ret
+diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
+index c4751ec..45e87c9 100644
+--- a/drivers/net/ethernet/broadcom/bgmac.c
++++ b/drivers/net/ethernet/broadcom/bgmac.c
+@@ -307,6 +307,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
+ 	u32 ctl;
+ 
+ 	ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL);
++
++	/* preserve ONLY bits 16-17 from current hardware value */
++	ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
++
+ 	if (bgmac->feature_flags & BGMAC_FEAT_RX_MASK_SETUP) {
+ 		ctl &= ~BGMAC_DMA_RX_BL_MASK;
+ 		ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT;
+@@ -317,7 +321,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac,
+ 		ctl &= ~BGMAC_DMA_RX_PT_MASK;
+ 		ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT;
+ 	}
+-	ctl &= BGMAC_DMA_RX_ADDREXT_MASK;
+ 	ctl |= BGMAC_DMA_RX_ENABLE;
+ 	ctl |= BGMAC_DMA_RX_PARITY_DISABLE;
+ 	ctl |= BGMAC_DMA_RX_OVERFLOW_CONT;
+diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
+index 505ceaf..2c850a9 100644
+--- a/drivers/net/ethernet/broadcom/bnx2.c
++++ b/drivers/net/ethernet/broadcom/bnx2.c
+@@ -49,6 +49,7 @@
+ #include <linux/firmware.h>
+ #include <linux/log2.h>
+ #include <linux/aer.h>
++#include <linux/crash_dump.h>
+ 
+ #if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE)
+ #define BCM_CNIC 1
+@@ -4759,15 +4760,16 @@ bnx2_setup_msix_tbl(struct bnx2 *bp)
+ 	BNX2_WR(bp, BNX2_PCI_GRC_WINDOW3_ADDR, BNX2_MSIX_PBA_ADDR);
+ }
+ 
+-static int
+-bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
++static void
++bnx2_wait_dma_complete(struct bnx2 *bp)
+ {
+ 	u32 val;
+-	int i, rc = 0;
+-	u8 old_port;
++	int i;
+ 
+-	/* Wait for the current PCI transaction to complete before
+-	 * issuing a reset. */
++	/*
++	 * Wait for the current PCI transaction to complete before
++	 * issuing a reset.
++	 */
+ 	if ((BNX2_CHIP(bp) == BNX2_CHIP_5706) ||
+ 	    (BNX2_CHIP(bp) == BNX2_CHIP_5708)) {
+ 		BNX2_WR(bp, BNX2_MISC_ENABLE_CLR_BITS,
+@@ -4791,6 +4793,21 @@ bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
+ 		}
+ 	}
+ 
++	return;
++}
++
++
++static int
++bnx2_reset_chip(struct bnx2 *bp, u32 reset_code)
++{
++	u32 val;
++	int i, rc = 0;
++	u8 old_port;
++
++	/* Wait for the current PCI transaction to complete before
++	 * issuing a reset. */
++	bnx2_wait_dma_complete(bp);
++
+ 	/* Wait for the firmware to tell us it is ok to issue a reset. */
+ 	bnx2_fw_sync(bp, BNX2_DRV_MSG_DATA_WAIT0 | reset_code, 1, 1);
+ 
+@@ -6356,6 +6373,10 @@ bnx2_open(struct net_device *dev)
+ 	struct bnx2 *bp = netdev_priv(dev);
+ 	int rc;
+ 
++	rc = bnx2_request_firmware(bp);
++	if (rc < 0)
++		goto out;
++
+ 	netif_carrier_off(dev);
+ 
+ 	bnx2_disable_int(bp);
+@@ -6424,6 +6445,7 @@ bnx2_open(struct net_device *dev)
+ 	bnx2_free_irq(bp);
+ 	bnx2_free_mem(bp);
+ 	bnx2_del_napi(bp);
++	bnx2_release_firmware(bp);
+ 	goto out;
+ }
+ 
+@@ -8570,12 +8592,15 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+ 
+ 	pci_set_drvdata(pdev, dev);
+ 
+-	rc = bnx2_request_firmware(bp);
+-	if (rc < 0)
+-		goto error;
+-
++	/*
++	 * In-flight DMA from 1st kernel could continue going in kdump kernel.
++	 * New io-page table has been created before bnx2 does reset at open stage.
++	 * We have to wait for the in-flight DMA to complete to avoid it look up
++	 * into the newly created io-page table.
++	 */
++	if (is_kdump_kernel())
++		bnx2_wait_dma_complete(bp);
+ 
+-	bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET);
+ 	memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN);
+ 
+ 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
+@@ -8608,7 +8633,6 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
+ 	return 0;
+ 
+ error:
+-	bnx2_release_firmware(bp);
+ 	pci_iounmap(pdev, bp->regview);
+ 	pci_release_regions(pdev);
+ 	pci_disable_device(pdev);
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+index d48873b..5cdc96b 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+@@ -231,7 +231,7 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp_port *port)
+ 
+ 	span_entry->used = true;
+ 	span_entry->id = index;
+-	span_entry->ref_count = 0;
++	span_entry->ref_count = 1;
+ 	span_entry->local_port = local_port;
+ 	return span_entry;
+ }
+@@ -268,6 +268,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
+ 
+ 	span_entry = mlxsw_sp_span_entry_find(port);
+ 	if (span_entry) {
++		/* Already exists, just take a reference */
+ 		span_entry->ref_count++;
+ 		return span_entry;
+ 	}
+@@ -278,6 +279,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port)
+ static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp,
+ 				   struct mlxsw_sp_span_entry *span_entry)
+ {
++	WARN_ON(!span_entry->ref_count);
+ 	if (--span_entry->ref_count == 0)
+ 		mlxsw_sp_span_entry_destroy(mlxsw_sp, span_entry);
+ 	return 0;
+diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+index 3f5c51d..62514b9 100644
+--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
++++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+@@ -777,6 +777,26 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
+ 	}
+ }
+ 
++static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
++{
++	u8 num_rec, last_rec_index, num_entries;
++
++	num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
++	last_rec_index = num_rec - 1;
++
++	if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
++		return false;
++	if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
++	    MLXSW_REG_RAUHTD_TYPE_IPV6)
++		return true;
++
++	num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
++								last_rec_index);
++	if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
++		return true;
++	return false;
++}
++
+ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
+ {
+ 	char *rauhtd_pl;
+@@ -803,7 +823,7 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
+ 		for (i = 0; i < num_rec; i++)
+ 			mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
+ 							  i);
+-	} while (num_rec);
++	} while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
+ 	rtnl_unlock();
+ 
+ 	kfree(rauhtd_pl);
+diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+index 4c8c60a..fe9e7b1 100644
+--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
++++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+@@ -871,6 +871,13 @@ static int stmmac_init_phy(struct net_device *dev)
+ 		return -ENODEV;
+ 	}
+ 
++	/* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid
++	 * subsequent PHY polling, make sure we force a link transition if
++	 * we have a UP/DOWN/UP transition
++	 */
++	if (phydev->is_pseudo_fixed_link)
++		phydev->irq = PHY_POLL;
++
+ 	pr_debug("stmmac_init_phy:  %s: attached to PHY (UID 0x%x)"
+ 		 " Link = %d\n", dev->name, phydev->phy_id, phydev->link);
+ 
+diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
+index 5c8429f..3a5530d 100644
+--- a/drivers/usb/gadget/function/f_fs.c
++++ b/drivers/usb/gadget/function/f_fs.c
+@@ -133,8 +133,60 @@ struct ffs_epfile {
+ 	/*
+ 	 * Buffer for holding data from partial reads which may happen since
+ 	 * we’re rounding user read requests to a multiple of a max packet size.
++	 *
++	 * The pointer is initialised with NULL value and may be set by
++	 * __ffs_epfile_read_data function to point to a temporary buffer.
++	 *
++	 * In normal operation, calls to __ffs_epfile_read_buffered will consume
++	 * data from said buffer and eventually free it.  Importantly, while the
++	 * function is using the buffer, it sets the pointer to NULL.  This is
++	 * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered
++	 * can never run concurrently (they are synchronised by epfile->mutex)
++	 * so the latter will not assign a new value to the pointer.
++	 *
++	 * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is
++	 * valid) and sets the pointer to READ_BUFFER_DROP value.  This special
++	 * value is crux of the synchronisation between ffs_func_eps_disable and
++	 * __ffs_epfile_read_data.
++	 *
++	 * Once __ffs_epfile_read_data is about to finish it will try to set the
++	 * pointer back to its old value (as described above), but seeing as the
++	 * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free
++	 * the buffer.
++	 *
++	 * == State transitions ==
++	 *
++	 * • ptr == NULL:  (initial state)
++	 *   ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP
++	 *   ◦ __ffs_epfile_read_buffered:    nop
++	 *   ◦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf
++	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
++	 * • ptr == DROP:
++	 *   ◦ __ffs_epfile_read_buffer_free: nop
++	 *   ◦ __ffs_epfile_read_buffered:    go to ptr == NULL
++	 *   ◦ __ffs_epfile_read_data allocates temp buffer: free buf, nop
++	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
++	 * • ptr == buf:
++	 *   ◦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP
++	 *   ◦ __ffs_epfile_read_buffered:    go to ptr == NULL and reading
++	 *   ◦ __ffs_epfile_read_data:        n/a, __ffs_epfile_read_buffered
++	 *                                    is always called first
++	 *   ◦ reading finishes:              n/a, not in ‘and reading’ state
++	 * • ptr == NULL and reading:
++	 *   ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading
++	 *   ◦ __ffs_epfile_read_buffered:    n/a, mutex is held
++	 *   ◦ __ffs_epfile_read_data:        n/a, mutex is held
++	 *   ◦ reading finishes and …
++	 *     … all data read:               free buf, go to ptr == NULL
++	 *     … otherwise:                   go to ptr == buf and reading
++	 * • ptr == DROP and reading:
++	 *   ◦ __ffs_epfile_read_buffer_free: nop
++	 *   ◦ __ffs_epfile_read_buffered:    n/a, mutex is held
++	 *   ◦ __ffs_epfile_read_data:        n/a, mutex is held
++	 *   ◦ reading finishes:              free buf, go to ptr == DROP
+ 	 */
+-	struct ffs_buffer		*read_buffer;	/* P: epfile->mutex */
++	struct ffs_buffer		*read_buffer;
++#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN))
+ 
+ 	char				name[5];
+ 
+@@ -733,25 +785,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep,
+ 	schedule_work(&io_data->work);
+ }
+ 
++static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile)
++{
++	/*
++	 * See comment in struct ffs_epfile for full read_buffer pointer
++	 * synchronisation story.
++	 */
++	struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP);
++	if (buf && buf != READ_BUFFER_DROP)
++		kfree(buf);
++}
++
+ /* Assumes epfile->mutex is held. */
+ static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile,
+ 					  struct iov_iter *iter)
+ {
+-	struct ffs_buffer *buf = epfile->read_buffer;
++	/*
++	 * Null out epfile->read_buffer so ffs_func_eps_disable does not free
++	 * the buffer while we are using it.  See comment in struct ffs_epfile
++	 * for full read_buffer pointer synchronisation story.
++	 */
++	struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL);
+ 	ssize_t ret;
+-	if (!buf)
++	if (!buf || buf == READ_BUFFER_DROP)
+ 		return 0;
+ 
+ 	ret = copy_to_iter(buf->data, buf->length, iter);
+ 	if (buf->length == ret) {
+ 		kfree(buf);
+-		epfile->read_buffer = NULL;
+-	} else if (unlikely(iov_iter_count(iter))) {
++		return ret;
++	}
++
++	if (unlikely(iov_iter_count(iter))) {
+ 		ret = -EFAULT;
+ 	} else {
+ 		buf->length -= ret;
+ 		buf->data += ret;
+ 	}
++
++	if (cmpxchg(&epfile->read_buffer, NULL, buf))
++		kfree(buf);
++
+ 	return ret;
+ }
+ 
+@@ -780,7 +854,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile,
+ 	buf->length = data_len;
+ 	buf->data = buf->storage;
+ 	memcpy(buf->storage, data + ret, data_len);
+-	epfile->read_buffer = buf;
++
++	/*
++	 * At this point read_buffer is NULL or READ_BUFFER_DROP (if
++	 * ffs_func_eps_disable has been called in the meanwhile).  See comment
++	 * in struct ffs_epfile for full read_buffer pointer synchronisation
++	 * story.
++	 */
++	if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf)))
++		kfree(buf);
+ 
+ 	return ret;
+ }
+@@ -1094,8 +1176,7 @@ ffs_epfile_release(struct inode *inode, struct file *file)
+ 
+ 	ENTER();
+ 
+-	kfree(epfile->read_buffer);
+-	epfile->read_buffer = NULL;
++	__ffs_epfile_read_buffer_free(epfile);
+ 	ffs_data_closed(epfile->ffs);
+ 
+ 	return 0;
+@@ -1721,24 +1802,20 @@ static void ffs_func_eps_disable(struct ffs_function *func)
+ 	unsigned count            = func->ffs->eps_count;
+ 	unsigned long flags;
+ 
++	spin_lock_irqsave(&func->ffs->eps_lock, flags);
+ 	do {
+-		if (epfile)
+-			mutex_lock(&epfile->mutex);
+-		spin_lock_irqsave(&func->ffs->eps_lock, flags);
+ 		/* pending requests get nuked */
+ 		if (likely(ep->ep))
+ 			usb_ep_disable(ep->ep);
+ 		++ep;
+-		spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
+ 
+ 		if (epfile) {
+ 			epfile->ep = NULL;
+-			kfree(epfile->read_buffer);
+-			epfile->read_buffer = NULL;
+-			mutex_unlock(&epfile->mutex);
++			__ffs_epfile_read_buffer_free(epfile);
+ 			++epfile;
+ 		}
+ 	} while (--count);
++	spin_unlock_irqrestore(&func->ffs->eps_lock, flags);
+ }
+ 
+ static int ffs_func_eps_enable(struct ffs_function *func)
+diff --git a/include/net/ip.h b/include/net/ip.h
+index 156b0c1..0ccf6da 100644
+--- a/include/net/ip.h
++++ b/include/net/ip.h
+@@ -47,7 +47,6 @@ struct inet_skb_parm {
+ #define IPSKB_REROUTED		BIT(4)
+ #define IPSKB_DOREDIRECT	BIT(5)
+ #define IPSKB_FRAG_PMTU		BIT(6)
+-#define IPSKB_FRAG_SEGS		BIT(7)
+ 
+ 	u16			frag_max_size;
+ };
+diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h
+index 43a5a0e..b01d5d1 100644
+--- a/include/net/ip6_tunnel.h
++++ b/include/net/ip6_tunnel.h
+@@ -145,6 +145,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb,
+ {
+ 	int pkt_len, err;
+ 
++	memset(skb->cb, 0, sizeof(struct inet6_skb_parm));
+ 	pkt_len = skb->len - skb_inner_network_offset(skb);
+ 	err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb);
+ 	if (unlikely(net_xmit_eval(err)))
+diff --git a/include/net/sock.h b/include/net/sock.h
+index 8741988..c26eab9 100644
+--- a/include/net/sock.h
++++ b/include/net/sock.h
+@@ -1587,11 +1587,11 @@ static inline void sock_put(struct sock *sk)
+ void sock_gen_put(struct sock *sk);
+ 
+ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
+-		     unsigned int trim_cap);
++		     unsigned int trim_cap, bool refcounted);
+ static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
+ 				 const int nested)
+ {
+-	return __sk_receive_skb(sk, skb, nested, 1);
++	return __sk_receive_skb(sk, skb, nested, 1, true);
+ }
+ 
+ static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
+diff --git a/include/net/tcp.h b/include/net/tcp.h
+index 7717302..0de6989 100644
+--- a/include/net/tcp.h
++++ b/include/net/tcp.h
+@@ -1164,6 +1164,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp)
+ }
+ 
+ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb);
++int tcp_filter(struct sock *sk, struct sk_buff *skb);
+ 
+ #undef STATE_TRACE
+ 
+diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h
+index 5cd4d4d..9c9c6ad 100644
+--- a/include/uapi/linux/atm_zatm.h
++++ b/include/uapi/linux/atm_zatm.h
+@@ -14,7 +14,6 @@
+ 
+ #include <linux/atmapi.h>
+ #include <linux/atmioc.h>
+-#include <linux/time.h>
+ 
+ #define ZATM_GETPOOL	_IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc)
+ 						/* get pool statistics */
+diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
+index 570eeca..ad1bc67 100644
+--- a/kernel/bpf/hashtab.c
++++ b/kernel/bpf/hashtab.c
+@@ -687,7 +687,8 @@ static void delete_all_elements(struct bpf_htab *htab)
+ 
+ 		hlist_for_each_entry_safe(l, n, head, hash_node) {
+ 			hlist_del_rcu(&l->hash_node);
+-			htab_elem_free(htab, l);
++			if (l->state != HTAB_EXTRA_ELEM_USED)
++				htab_elem_free(htab, l);
+ 		}
+ 	}
+ }
+diff --git a/net/core/dev.c b/net/core/dev.c
+index 44b3ba4..9ce9d72 100644
+--- a/net/core/dev.c
++++ b/net/core/dev.c
+@@ -2484,7 +2484,7 @@ int skb_checksum_help(struct sk_buff *skb)
+ 			goto out;
+ 	}
+ 
+-	*(__sum16 *)(skb->data + offset) = csum_fold(csum);
++	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
+ out_set_summed:
+ 	skb->ip_summed = CHECKSUM_NONE;
+ out:
+diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
+index 52742a0..5550a86 100644
+--- a/net/core/flow_dissector.c
++++ b/net/core/flow_dissector.c
+@@ -118,7 +118,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
+ 	struct flow_dissector_key_tags *key_tags;
+ 	struct flow_dissector_key_keyid *key_keyid;
+ 	u8 ip_proto = 0;
+-	bool ret = false;
++	bool ret;
+ 
+ 	if (!data) {
+ 		data = skb->data;
+@@ -481,12 +481,17 @@ bool __skb_flow_dissect(const struct sk_buff *skb,
+ out_good:
+ 	ret = true;
+ 
+-out_bad:
++	key_control->thoff = (u16)nhoff;
++out:
+ 	key_basic->n_proto = proto;
+ 	key_basic->ip_proto = ip_proto;
+-	key_control->thoff = (u16)nhoff;
+ 
+ 	return ret;
++
++out_bad:
++	ret = false;
++	key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen);
++	goto out;
+ }
+ EXPORT_SYMBOL(__skb_flow_dissect);
+ 
+diff --git a/net/core/sock.c b/net/core/sock.c
+index fd7b41e..10acacc 100644
+--- a/net/core/sock.c
++++ b/net/core/sock.c
+@@ -453,7 +453,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
+ EXPORT_SYMBOL(sock_queue_rcv_skb);
+ 
+ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
+-		     const int nested, unsigned int trim_cap)
++		     const int nested, unsigned int trim_cap, bool refcounted)
+ {
+ 	int rc = NET_RX_SUCCESS;
+ 
+@@ -487,7 +487,8 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
+ 
+ 	bh_unlock_sock(sk);
+ out:
+-	sock_put(sk);
++	if (refcounted)
++		sock_put(sk);
+ 	return rc;
+ discard_and_relse:
+ 	kfree_skb(skb);
+@@ -1563,6 +1564,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
+ 		RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
+ 
+ 		newsk->sk_err	   = 0;
++		newsk->sk_err_soft = 0;
+ 		newsk->sk_priority = 0;
+ 		newsk->sk_incoming_cpu = raw_smp_processor_id();
+ 		atomic64_set(&newsk->sk_cookie, 0);
+diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
+index 345a3ae..b567c87 100644
+--- a/net/dccp/ipv4.c
++++ b/net/dccp/ipv4.c
+@@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
+ {
+ 	const struct iphdr *iph = (struct iphdr *)skb->data;
+ 	const u8 offset = iph->ihl << 2;
+-	const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
++	const struct dccp_hdr *dh;
+ 	struct dccp_sock *dp;
+ 	struct inet_sock *inet;
+ 	const int type = icmp_hdr(skb)->type;
+@@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info)
+ 	int err;
+ 	struct net *net = dev_net(skb->dev);
+ 
+-	if (skb->len < offset + sizeof(*dh) ||
+-	    skb->len < offset + __dccp_basic_hdr_len(dh)) {
+-		__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
+-		return;
+-	}
++	/* Only need dccph_dport & dccph_sport which are the first
++	 * 4 bytes in dccp header.
++	 * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us.
++	 */
++	BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
++	BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
++	dh = (struct dccp_hdr *)(skb->data + offset);
+ 
+ 	sk = __inet_lookup_established(net, &dccp_hashinfo,
+ 				       iph->daddr, dh->dccph_dport,
+@@ -868,7 +870,7 @@ static int dccp_v4_rcv(struct sk_buff *skb)
+ 		goto discard_and_relse;
+ 	nf_reset(skb);
+ 
+-	return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4);
++	return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, refcounted);
+ 
+ no_dccp_socket:
+ 	if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
+diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
+index 3828f94..715e5d1 100644
+--- a/net/dccp/ipv6.c
++++ b/net/dccp/ipv6.c
+@@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
+ 			u8 type, u8 code, int offset, __be32 info)
+ {
+ 	const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data;
+-	const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset);
++	const struct dccp_hdr *dh;
+ 	struct dccp_sock *dp;
+ 	struct ipv6_pinfo *np;
+ 	struct sock *sk;
+@@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
+ 	__u64 seq;
+ 	struct net *net = dev_net(skb->dev);
+ 
+-	if (skb->len < offset + sizeof(*dh) ||
+-	    skb->len < offset + __dccp_basic_hdr_len(dh)) {
+-		__ICMP6_INC_STATS(net, __in6_dev_get(skb->dev),
+-				  ICMP6_MIB_INERRORS);
+-		return;
+-	}
++	/* Only need dccph_dport & dccph_sport which are the first
++	 * 4 bytes in dccp header.
++	 * Our caller (icmpv6_notify()) already pulled 8 bytes for us.
++	 */
++	BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8);
++	BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8);
++	dh = (struct dccp_hdr *)(skb->data + offset);
+ 
+ 	sk = __inet6_lookup_established(net, &dccp_hashinfo,
+ 					&hdr->daddr, dh->dccph_dport,
+@@ -738,7 +739,8 @@ static int dccp_v6_rcv(struct sk_buff *skb)
+ 	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
+ 		goto discard_and_relse;
+ 
+-	return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4) ? -1 : 0;
++	return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4,
++				refcounted) ? -1 : 0;
+ 
+ no_dccp_socket:
+ 	if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb))
+@@ -956,6 +958,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = {
+ 	.getsockopt	   = ipv6_getsockopt,
+ 	.addr2sockaddr	   = inet6_csk_addr2sockaddr,
+ 	.sockaddr_len	   = sizeof(struct sockaddr_in6),
++	.bind_conflict	   = inet6_csk_bind_conflict,
+ #ifdef CONFIG_COMPAT
+ 	.compat_setsockopt = compat_ipv6_setsockopt,
+ 	.compat_getsockopt = compat_ipv6_getsockopt,
+diff --git a/net/dccp/proto.c b/net/dccp/proto.c
+index 41e6580..9fe25bf 100644
+--- a/net/dccp/proto.c
++++ b/net/dccp/proto.c
+@@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout)
+ 		__kfree_skb(skb);
+ 	}
+ 
++	/* If socket has been already reset kill it. */
++	if (sk->sk_state == DCCP_CLOSED)
++		goto adjudge_to_death;
++
+ 	if (data_was_unread) {
+ 		/* Unread data was tossed, send an appropriate Reset Code */
+ 		DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread);
+diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c
+index e2ffc2a..7ef7031 100644
+--- a/net/ipv4/fib_trie.c
++++ b/net/ipv4/fib_trie.c
+@@ -2455,22 +2455,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
+ 	struct key_vector *l, **tp = &iter->tnode;
+ 	t_key key;
+ 
+-	/* use cache location of next-to-find key */
++	/* use cached location of previously found key */
+ 	if (iter->pos > 0 && pos >= iter->pos) {
+-		pos -= iter->pos;
+ 		key = iter->key;
+ 	} else {
+-		iter->pos = 0;
++		iter->pos = 1;
+ 		key = 0;
+ 	}
+ 
+-	while ((l = leaf_walk_rcu(tp, key)) != NULL) {
++	pos -= iter->pos;
++
++	while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) {
+ 		key = l->key + 1;
+ 		iter->pos++;
+-
+-		if (--pos <= 0)
+-			break;
+-
+ 		l = NULL;
+ 
+ 		/* handle unlikely case of a key wrap */
+@@ -2479,7 +2476,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter,
+ 	}
+ 
+ 	if (l)
+-		iter->key = key;	/* remember it */
++		iter->key = l->key;	/* remember it */
+ 	else
+ 		iter->pos = 0;		/* forget it */
+ 
+@@ -2507,7 +2504,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos)
+ 		return fib_route_get_idx(iter, *pos);
+ 
+ 	iter->pos = 0;
+-	iter->key = 0;
++	iter->key = KEY_MAX;
+ 
+ 	return SEQ_START_TOKEN;
+ }
+@@ -2516,7 +2513,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+ {
+ 	struct fib_route_iter *iter = seq->private;
+ 	struct key_vector *l = NULL;
+-	t_key key = iter->key;
++	t_key key = iter->key + 1;
+ 
+ 	++*pos;
+ 
+@@ -2525,7 +2522,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+ 		l = leaf_walk_rcu(&iter->tnode, key);
+ 
+ 	if (l) {
+-		iter->key = l->key + 1;
++		iter->key = l->key;
+ 		iter->pos++;
+ 	} else {
+ 		iter->pos = 0;
+diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
+index 38abe70..48734ee 100644
+--- a/net/ipv4/icmp.c
++++ b/net/ipv4/icmp.c
+@@ -477,7 +477,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
+ 	fl4->flowi4_proto = IPPROTO_ICMP;
+ 	fl4->fl4_icmp_type = type;
+ 	fl4->fl4_icmp_code = code;
+-	fl4->flowi4_oif = l3mdev_master_ifindex(skb_in->dev);
++	fl4->flowi4_oif = l3mdev_master_ifindex(skb_dst(skb_in)->dev);
+ 
+ 	security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4));
+ 	rt = __ip_route_output_key_hash(net, fl4,
+@@ -502,7 +502,7 @@ static struct rtable *icmp_route_lookup(struct net *net,
+ 	if (err)
+ 		goto relookup_failed;
+ 
+-	if (inet_addr_type_dev_table(net, skb_in->dev,
++	if (inet_addr_type_dev_table(net, skb_dst(skb_in)->dev,
+ 				     fl4_dec.saddr) == RTN_LOCAL) {
+ 		rt2 = __ip_route_output_key(net, &fl4_dec);
+ 		if (IS_ERR(rt2))
+diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c
+index 8b4ffd2..9f0a7b9 100644
+--- a/net/ipv4/ip_forward.c
++++ b/net/ipv4/ip_forward.c
+@@ -117,7 +117,7 @@ int ip_forward(struct sk_buff *skb)
+ 	if (opt->is_strictroute && rt->rt_uses_gateway)
+ 		goto sr_failed;
+ 
+-	IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
++	IPCB(skb)->flags |= IPSKB_FORWARDED;
+ 	mtu = ip_dst_mtu_maybe_forward(&rt->dst, true);
+ 	if (ip_exceeds_mtu(skb, mtu)) {
+ 		IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS);
+diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
+index dde37fb..307daed 100644
+--- a/net/ipv4/ip_output.c
++++ b/net/ipv4/ip_output.c
+@@ -223,11 +223,9 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk,
+ 	struct sk_buff *segs;
+ 	int ret = 0;
+ 
+-	/* common case: fragmentation of segments is not allowed,
+-	 * or seglen is <= mtu
++	/* common case: seglen is <= mtu
+ 	 */
+-	if (((IPCB(skb)->flags & IPSKB_FRAG_SEGS) == 0) ||
+-	      skb_gso_validate_mtu(skb, mtu))
++	if (skb_gso_validate_mtu(skb, mtu))
+ 		return ip_finish_output2(net, sk, skb);
+ 
+ 	/* Slowpath -  GSO segment length is exceeding the dst MTU.
+diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
+index 0f227db..afd6b59 100644
+--- a/net/ipv4/ip_tunnel_core.c
++++ b/net/ipv4/ip_tunnel_core.c
+@@ -63,7 +63,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
+ 	int pkt_len = skb->len - skb_inner_network_offset(skb);
+ 	struct net *net = dev_net(rt->dst.dev);
+ 	struct net_device *dev = skb->dev;
+-	int skb_iif = skb->skb_iif;
+ 	struct iphdr *iph;
+ 	int err;
+ 
+@@ -73,16 +72,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb,
+ 	skb_dst_set(skb, &rt->dst);
+ 	memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
+ 
+-	if (skb_iif && !(df & htons(IP_DF))) {
+-		/* Arrived from an ingress interface, got encapsulated, with
+-		 * fragmentation of encapulating frames allowed.
+-		 * If skb is gso, the resulting encapsulated network segments
+-		 * may exceed dst mtu.
+-		 * Allow IP Fragmentation of segments.
+-		 */
+-		IPCB(skb)->flags |= IPSKB_FRAG_SEGS;
+-	}
+-
+ 	/* Push down and install the IP header. */
+ 	skb_push(skb, sizeof(struct iphdr));
+ 	skb_reset_network_header(skb);
+diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
+index 5f006e1..27089f5 100644
+--- a/net/ipv4/ipmr.c
++++ b/net/ipv4/ipmr.c
+@@ -1749,7 +1749,7 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt,
+ 		vif->dev->stats.tx_bytes += skb->len;
+ 	}
+ 
+-	IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS;
++	IPCB(skb)->flags |= IPSKB_FORWARDED;
+ 
+ 	/* RFC1584 teaches, that DVMRP/PIM router must deliver packets locally
+ 	 * not only before forwarding, but after forwarding on all output
+diff --git a/net/ipv4/route.c b/net/ipv4/route.c
+index 62c3ed0..2f23ef1 100644
+--- a/net/ipv4/route.c
++++ b/net/ipv4/route.c
+@@ -753,7 +753,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow
+ 			goto reject_redirect;
+ 	}
+ 
+-	n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw);
++	n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
++	if (!n)
++		n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
+ 	if (!IS_ERR(n)) {
+ 		if (!(n->nud_state & NUD_VALID)) {
+ 			neigh_event_send(n, NULL);
+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
+index ffbb218..c876f5d 100644
+--- a/net/ipv4/tcp.c
++++ b/net/ipv4/tcp.c
+@@ -1145,7 +1145,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
+ 
+ 	err = -EPIPE;
+ 	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
+-		goto out_err;
++		goto do_error;
+ 
+ 	sg = !!(sk->sk_route_caps & NETIF_F_SG);
+ 
+@@ -1219,7 +1219,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
+ 
+ 			if (!skb_can_coalesce(skb, i, pfrag->page,
+ 					      pfrag->offset)) {
+-				if (i == sysctl_max_skb_frags || !sg) {
++				if (i >= sysctl_max_skb_frags || !sg) {
+ 					tcp_mark_push(tp, skb);
+ 					goto new_segment;
+ 				}
+diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c
+index 10d728b..ab37c67 100644
+--- a/net/ipv4/tcp_dctcp.c
++++ b/net/ipv4/tcp_dctcp.c
+@@ -56,6 +56,7 @@ struct dctcp {
+ 	u32 next_seq;
+ 	u32 ce_state;
+ 	u32 delayed_ack_reserved;
++	u32 loss_cwnd;
+ };
+ 
+ static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */
+@@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk)
+ 		ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
+ 
+ 		ca->delayed_ack_reserved = 0;
++		ca->loss_cwnd = 0;
+ 		ca->ce_state = 0;
+ 
+ 		dctcp_reset(tp, ca);
+@@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk)
+ 
+ static u32 dctcp_ssthresh(struct sock *sk)
+ {
+-	const struct dctcp *ca = inet_csk_ca(sk);
++	struct dctcp *ca = inet_csk_ca(sk);
+ 	struct tcp_sock *tp = tcp_sk(sk);
+ 
++	ca->loss_cwnd = tp->snd_cwnd;
+ 	return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
+ }
+ 
+@@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr,
+ 	return 0;
+ }
+ 
++static u32 dctcp_cwnd_undo(struct sock *sk)
++{
++	const struct dctcp *ca = inet_csk_ca(sk);
++
++	return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
++}
++
+ static struct tcp_congestion_ops dctcp __read_mostly = {
+ 	.init		= dctcp_init,
+ 	.in_ack_event   = dctcp_update_alpha,
+ 	.cwnd_event	= dctcp_cwnd_event,
+ 	.ssthresh	= dctcp_ssthresh,
+ 	.cong_avoid	= tcp_reno_cong_avoid,
++	.undo_cwnd	= dctcp_cwnd_undo,
+ 	.set_state	= dctcp_state,
+ 	.get_info	= dctcp_get_info,
+ 	.flags		= TCP_CONG_NEEDS_ECN,
+diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
+index 7158d4f..7b235fa 100644
+--- a/net/ipv4/tcp_ipv4.c
++++ b/net/ipv4/tcp_ipv4.c
+@@ -1537,6 +1537,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb)
+ }
+ EXPORT_SYMBOL(tcp_prequeue);
+ 
++int tcp_filter(struct sock *sk, struct sk_buff *skb)
++{
++	struct tcphdr *th = (struct tcphdr *)skb->data;
++	unsigned int eaten = skb->len;
++	int err;
++
++	err = sk_filter_trim_cap(sk, skb, th->doff * 4);
++	if (!err) {
++		eaten -= skb->len;
++		TCP_SKB_CB(skb)->end_seq -= eaten;
++	}
++	return err;
++}
++EXPORT_SYMBOL(tcp_filter);
++
+ /*
+  *	From tcp_input.c
+  */
+@@ -1648,8 +1663,10 @@ int tcp_v4_rcv(struct sk_buff *skb)
+ 
+ 	nf_reset(skb);
+ 
+-	if (sk_filter(sk, skb))
++	if (tcp_filter(sk, skb))
+ 		goto discard_and_relse;
++	th = (const struct tcphdr *)skb->data;
++	iph = ip_hdr(skb);
+ 
+ 	skb->dev = NULL;
+ 
+diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
+index bd59c34..7370ad2 100644
+--- a/net/ipv6/icmp.c
++++ b/net/ipv6/icmp.c
+@@ -448,7 +448,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info,
+ 	if (__ipv6_addr_needs_scope_id(addr_type))
+ 		iif = skb->dev->ifindex;
+ 	else
+-		iif = l3mdev_master_ifindex(skb->dev);
++		iif = l3mdev_master_ifindex(skb_dst(skb)->dev);
+ 
+ 	/*
+ 	 *	Must not send error if the source does not uniquely
+diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
+index fc67822..af6a09e 100644
+--- a/net/ipv6/tcp_ipv6.c
++++ b/net/ipv6/tcp_ipv6.c
+@@ -1228,7 +1228,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
+ 	if (skb->protocol == htons(ETH_P_IP))
+ 		return tcp_v4_do_rcv(sk, skb);
+ 
+-	if (sk_filter(sk, skb))
++	if (tcp_filter(sk, skb))
+ 		goto discard;
+ 
+ 	/*
+@@ -1455,8 +1455,10 @@ static int tcp_v6_rcv(struct sk_buff *skb)
+ 	if (tcp_v6_inbound_md5_hash(sk, skb))
+ 		goto discard_and_relse;
+ 
+-	if (sk_filter(sk, skb))
++	if (tcp_filter(sk, skb))
+ 		goto discard_and_relse;
++	th = (const struct tcphdr *)skb->data;
++	hdr = ipv6_hdr(skb);
+ 
+ 	skb->dev = NULL;
+ 
+diff --git a/net/sctp/socket.c b/net/sctp/socket.c
+index baccbf3..7b0e059 100644
+--- a/net/sctp/socket.c
++++ b/net/sctp/socket.c
+@@ -1214,9 +1214,12 @@ static int __sctp_connect(struct sock *sk,
+ 
+ 	timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK);
+ 
+-	err = sctp_wait_for_connect(asoc, &timeo);
+-	if ((err == 0 || err == -EINPROGRESS) && assoc_id)
++	if (assoc_id)
+ 		*assoc_id = asoc->assoc_id;
++	err = sctp_wait_for_connect(asoc, &timeo);
++	/* Note: the asoc may be freed after the return of
++	 * sctp_wait_for_connect.
++	 */
+ 
+ 	/* Don't free association on exit. */
+ 	asoc = NULL;
+@@ -4278,19 +4281,18 @@ static void sctp_shutdown(struct sock *sk, int how)
+ {
+ 	struct net *net = sock_net(sk);
+ 	struct sctp_endpoint *ep;
+-	struct sctp_association *asoc;
+ 
+ 	if (!sctp_style(sk, TCP))
+ 		return;
+ 
+-	if (how & SEND_SHUTDOWN) {
++	ep = sctp_sk(sk)->ep;
++	if (how & SEND_SHUTDOWN && !list_empty(&ep->asocs)) {
++		struct sctp_association *asoc;
++
+ 		sk->sk_state = SCTP_SS_CLOSING;
+-		ep = sctp_sk(sk)->ep;
+-		if (!list_empty(&ep->asocs)) {
+-			asoc = list_entry(ep->asocs.next,
+-					  struct sctp_association, asocs);
+-			sctp_primitive_SHUTDOWN(net, asoc, NULL);
+-		}
++		asoc = list_entry(ep->asocs.next,
++				  struct sctp_association, asocs);
++		sctp_primitive_SHUTDOWN(net, asoc, NULL);
+ 	}
+ }
+ 
+diff --git a/net/socket.c b/net/socket.c
+index a1bd161..03bc2c2 100644
+--- a/net/socket.c
++++ b/net/socket.c
+@@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen,
+ 		if (err)
+ 			break;
+ 		++datagrams;
++		if (msg_data_left(&msg_sys))
++			break;
+ 		cond_resched();
+ 	}
+ 
+diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c
+index f3825b6..f046b77 100644
+--- a/tools/spi/spidev_test.c
++++ b/tools/spi/spidev_test.c
+@@ -19,6 +19,7 @@
+ #include <getopt.h>
+ #include <fcntl.h>
+ #include <sys/ioctl.h>
++#include <linux/ioctl.h>
+ #include <sys/stat.h>
+ #include <linux/types.h>
+ #include <linux/spi/spidev.h>
diff --git a/4.8.9/4420_grsecurity-3.1-4.8.9-201611192033.patch b/4.8.10/4420_grsecurity-3.1-4.8.10-201611232213.patch
index f8ce60c..0149d93 100644
--- a/4.8.9/4420_grsecurity-3.1-4.8.9-201611192033.patch
+++ b/4.8.10/4420_grsecurity-3.1-4.8.10-201611232213.patch
@@ -407,7 +407,7 @@ index ffab8b5..b8fcd61 100644
  
  A toggle value indicating if modules are allowed to be loaded
 diff --git a/Makefile b/Makefile
-index c1519ab..90f06a0 100644
+index 7cf2b49..3e3071c 100644
 --- a/Makefile
 +++ b/Makefile
 @@ -302,7 +302,9 @@ CONFIG_SHELL := $(shell if [ -x "$$BASH" ]; then echo $$BASH; \
@@ -11039,7 +11039,7 @@ index ea55f86..dbf15cf 100644
  }
  
 diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
-index 37a315d..75ce910 100644
+index a6847fc..a5ac14a 100644
 --- a/arch/sparc/include/asm/uaccess_64.h
 +++ b/arch/sparc/include/asm/uaccess_64.h
 @@ -10,6 +10,7 @@
@@ -11062,26 +11062,26 @@ index 37a315d..75ce910 100644
  static inline int access_ok(int type, const void __user * addr, unsigned long size)
  {
  	return 1;
-@@ -212,6 +218,9 @@ copy_from_user(void *to, const void __user *from, unsigned long size)
+@@ -207,6 +213,9 @@ unsigned long __must_check ___copy_from_user(void *to,
+ static inline unsigned long __must_check
+ copy_from_user(void *to, const void __user *from, unsigned long size)
  {
- 	unsigned long ret;
- 
 +	if ((long)size < 0 || size > INT_MAX)
 +		return size;
 +
  	check_object_size(to, size, false);
  
- 	ret = ___copy_from_user(to, from, size);
-@@ -232,6 +241,9 @@ copy_to_user(void __user *to, const void *from, unsigned long size)
+ 	return ___copy_from_user(to, from, size);
+@@ -219,6 +228,9 @@ unsigned long __must_check ___copy_to_user(void __user *to,
+ static inline unsigned long __must_check
+ copy_to_user(void __user *to, const void *from, unsigned long size)
  {
- 	unsigned long ret;
- 
 +	if ((long)size < 0 || size > INT_MAX)
 +		return size;
 +
  	check_object_size(from, size, true);
  
- 	ret = ___copy_to_user(to, from, size);
+ 	return ___copy_to_user(to, from, size);
 diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
 index fdb1332..1b10f89 100644
 --- a/arch/sparc/kernel/Makefile
@@ -11732,7 +11732,7 @@ index 9aacb91..6415c82 100644
  	}
  }
 diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
-index 3269b02..64f5231 100644
+index 4f2384a..4e88949 100644
 --- a/arch/sparc/lib/Makefile
 +++ b/arch/sparc/lib/Makefile
 @@ -2,7 +2,7 @@
@@ -53086,7 +53086,7 @@ index 92e2243..8fd9092 100644
  		.ident = "Shift",
  		.matches = {
 diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
-index 822fc4a..24fd1f4 100644
+index 822fc4a..759e092 100644
 --- a/drivers/iommu/amd_iommu.c
 +++ b/drivers/iommu/amd_iommu.c
 @@ -791,11 +791,21 @@ static void copy_cmd_to_buffer(struct amd_iommu *iommu,
@@ -53113,6 +53113,119 @@ index 822fc4a..24fd1f4 100644
  	cmd->data[2] = 1;
  	CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
  }
+@@ -940,15 +950,13 @@ static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)
+  * Writes the command to the IOMMUs command buffer and informs the
+  * hardware about the new command.
+  */
+-static int iommu_queue_command_sync(struct amd_iommu *iommu,
+-				    struct iommu_cmd *cmd,
+-				    bool sync)
++static int __iommu_queue_command_sync(struct amd_iommu *iommu,
++				      struct iommu_cmd *cmd,
++				      bool sync)
+ {
+ 	u32 left, tail, head, next_tail;
+-	unsigned long flags;
+ 
+ again:
+-	spin_lock_irqsave(&iommu->lock, flags);
+ 
+ 	head      = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
+ 	tail      = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+@@ -957,15 +965,14 @@ again:
+ 
+ 	if (left <= 2) {
+ 		struct iommu_cmd sync_cmd;
+-		volatile u64 sem = 0;
+ 		int ret;
+ 
+-		build_completion_wait(&sync_cmd, (u64)&sem);
++		iommu->cmd_sem = 0;
++
++		build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
+ 		copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+ 
+-		spin_unlock_irqrestore(&iommu->lock, flags);
+-
+-		if ((ret = wait_on_sem(&sem)) != 0)
++		if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
+ 			return ret;
+ 
+ 		goto again;
+@@ -976,11 +983,23 @@ again:
+ 	/* We need to sync now to make sure all commands are processed */
+ 	iommu->need_sync = sync;
+ 
+-	spin_unlock_irqrestore(&iommu->lock, flags);
+-
+ 	return 0;
+ }
+ 
++static int iommu_queue_command_sync(struct amd_iommu *iommu,
++				    struct iommu_cmd *cmd,
++				    bool sync)
++{
++	unsigned long flags;
++	int ret;
++
++	spin_lock_irqsave(&iommu->lock, flags);
++	ret = __iommu_queue_command_sync(iommu, cmd, sync);
++	spin_unlock_irqrestore(&iommu->lock, flags);
++
++	return ret;
++}
++
+ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
+ {
+ 	return iommu_queue_command_sync(iommu, cmd, true);
+@@ -993,19 +1012,29 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
+ static int iommu_completion_wait(struct amd_iommu *iommu)
+ {
+ 	struct iommu_cmd cmd;
+-	volatile u64 sem = 0;
++	unsigned long flags;
+ 	int ret;
+ 
+ 	if (!iommu->need_sync)
+ 		return 0;
+ 
+-	build_completion_wait(&cmd, (u64)&sem);
+ 
+-	ret = iommu_queue_command_sync(iommu, &cmd, false);
++	build_completion_wait(&cmd, (u64)&iommu->cmd_sem);
++
++	spin_lock_irqsave(&iommu->lock, flags);
++
++	iommu->cmd_sem = 0;
++
++	ret = __iommu_queue_command_sync(iommu, &cmd, false);
+ 	if (ret)
+-		return ret;
++		goto out_unlock;
+ 
+-	return wait_on_sem(&sem);
++	ret = wait_on_sem(&iommu->cmd_sem);
++
++out_unlock:
++	spin_unlock_irqrestore(&iommu->lock, flags);
++
++	return ret;
+ }
+ 
+ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
+diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
+index caf5e38..9652848 100644
+--- a/drivers/iommu/amd_iommu_types.h
++++ b/drivers/iommu/amd_iommu_types.h
+@@ -524,6 +524,8 @@ struct amd_iommu {
+ 	struct irq_domain *ir_domain;
+ 	struct irq_domain *msi_domain;
+ #endif
++
++	volatile u64 __aligned(8) cmd_sem;
+ };
+ 
+ #define ACPIHID_UID_LEN 256
 diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
 index 641e887..df73c18 100644
 --- a/drivers/iommu/arm-smmu-v3.c
@@ -62705,10 +62818,10 @@ index 6c8bc5f..58c4f8c 100644
  	struct bcm_enet_priv *priv;
  	struct bcm_enet_desc *desc;
 diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
-index 505ceaf..c88cfa9 100644
+index 2c850a9..485b70c 100644
 --- a/drivers/net/ethernet/broadcom/bnx2.c
 +++ b/drivers/net/ethernet/broadcom/bnx2.c
-@@ -8703,7 +8703,7 @@ static SIMPLE_DEV_PM_OPS(bnx2_pm_ops, bnx2_suspend, bnx2_resume);
+@@ -8727,7 +8727,7 @@ static SIMPLE_DEV_PM_OPS(bnx2_pm_ops, bnx2_suspend, bnx2_resume);
   * this device has been detected.
   */
  static pci_ers_result_t bnx2_io_error_detected(struct pci_dev *pdev,
@@ -64496,10 +64609,10 @@ index 2385bae..5413c50 100644
  	struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
  	struct mlx5_priv *priv = &dev->priv;
 diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
-index d48873b..426f12e 100644
+index 5cdc96b..4c2a818 100644
 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
 +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
-@@ -4533,16 +4533,16 @@ static int mlxsw_sp_netdevice_event(struct notifier_block *unused,
+@@ -4535,16 +4535,16 @@ static int mlxsw_sp_netdevice_event(struct notifier_block *unused,
  	return notifier_from_errno(err);
  }
  
@@ -65022,10 +65135,10 @@ index ce9aa79..ad1831f 100644
  
  /* To mask all all interrupts.*/
 diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
-index 4c8c60a..c29928c 100644
+index fe9e7b1..55f1ba6 100644
 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
 +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
-@@ -1164,8 +1164,8 @@ static int alloc_dma_desc_resources(struct stmmac_priv *priv)
+@@ -1171,8 +1171,8 @@ static int alloc_dma_desc_resources(struct stmmac_priv *priv)
  	if (!priv->rx_skbuff)
  		goto err_rx_skbuff;
  
@@ -103246,20 +103359,22 @@ index 9dc4c6d..ed7c0e7 100644
  		EXOFS_ERR("read_cache_pages => %d\n", ret);
  		return ret;
 diff --git a/fs/exofs/super.c b/fs/exofs/super.c
-index 1076a42..54faf08 100644
+index 1076a42..40b7536 100644
 --- a/fs/exofs/super.c
 +++ b/fs/exofs/super.c
-@@ -192,10 +192,11 @@ static void exofs_init_once(void *foo)
+@@ -192,10 +192,12 @@ static void exofs_init_once(void *foo)
   */
  static int init_inodecache(void)
  {
 -	exofs_inode_cachep = kmem_cache_create("exofs_inode_cache",
 +	exofs_inode_cachep = kmem_cache_create_usercopy("exofs_inode_cache",
  				sizeof(struct exofs_i_info), 0,
- 				SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD |
+-				SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD |
 -				SLAB_ACCOUNT, exofs_init_once);
-+				SLAB_ACCOUNT, offsetof(struct exofs_i_info, i_data),
-+				sizeof(((struct exofs_i_info *)0)->i_data), exofs_init_once);
++				SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
++				offsetof(struct exofs_i_info, i_data),
++				sizeof(((struct exofs_i_info *)0)->i_data),
++				exofs_init_once);
  	if (exofs_inode_cachep == NULL)
  		return -ENOMEM;
  	return 0;
@@ -140820,10 +140935,10 @@ index 235c781..160d4a3 100644
  		struct rcu_head         rcu;
  		struct inet_peer	*gc_next;
 diff --git a/include/net/ip.h b/include/net/ip.h
-index 156b0c1..f921618 100644
+index 0ccf6da..5f0319d 100644
 --- a/include/net/ip.h
 +++ b/include/net/ip.h
-@@ -326,7 +326,7 @@ static inline unsigned int ip_skb_dst_mtu(struct sock *sk,
+@@ -325,7 +325,7 @@ static inline unsigned int ip_skb_dst_mtu(struct sock *sk,
  	return min(skb_dst(skb)->dev->mtu, IP_MAX_MTU);
  }
  
@@ -141454,7 +141569,7 @@ index c9228ad..5543dfb 100644
  #define SNMP_INC_STATS(mib, field)	\
  			this_cpu_inc(mib->mibs[field])
 diff --git a/include/net/sock.h b/include/net/sock.h
-index 8741988..ed2e15c 100644
+index c26eab9..7f31228 100644
 --- a/include/net/sock.h
 +++ b/include/net/sock.h
 @@ -188,7 +188,7 @@ struct sock_common {
@@ -141529,7 +141644,7 @@ index 8741988..ed2e15c 100644
  
  void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
 diff --git a/include/net/tcp.h b/include/net/tcp.h
-index 7717302..a633d63 100644
+index 0de6989..9846ad0 100644
 --- a/include/net/tcp.h
 +++ b/include/net/tcp.h
 @@ -543,7 +543,7 @@ void tcp_retransmit_timer(struct sock *sk);
@@ -141561,7 +141676,7 @@ index 7717302..a633d63 100644
  	union {
  		struct {
  			/* There is space for up to 20 bytes */
-@@ -1872,7 +1872,7 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
+@@ -1873,7 +1873,7 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb)
   */
  static inline void tcp_listendrop(const struct sock *sk)
  {
@@ -150151,6 +150266,45 @@ index 0bb9cf2..f319026 100644
  
  	key = event->type & (EVENT_HASHSIZE - 1);
  
+diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
+index ad1d616..139606f 100644
+--- a/kernel/trace/trace_printk.c
++++ b/kernel/trace/trace_printk.c
+@@ -167,6 +167,10 @@ static void format_mod_stop(void)
+ 	mutex_unlock(&btrace_mutex);
+ }
+ 
++static struct notifier_block module_trace_bprintk_format_nb = {
++	.notifier_call = module_trace_bprintk_format_notify,
++};
++
+ #else /* !CONFIG_MODULES */
+ __init static int
+ module_trace_bprintk_format_notify(struct notifier_block *self,
+@@ -190,11 +194,6 @@ void trace_printk_control(bool enabled)
+ 	trace_printk_enabled = enabled;
+ }
+ 
+-__initdata_or_module static
+-struct notifier_block module_trace_bprintk_format_nb = {
+-	.notifier_call = module_trace_bprintk_format_notify,
+-};
+-
+ int __trace_bprintk(unsigned long ip, const char *fmt, ...)
+  {
+ 	int ret;
+@@ -373,9 +372,11 @@ static __init int init_trace_printk_function_export(void)
+ 
+ fs_initcall(init_trace_printk_function_export);
+ 
++#ifdef CONFIG_MODULES
+ static __init int init_trace_printk(void)
+ {
+ 	return register_module_notifier(&module_trace_bprintk_format_nb);
+ }
+ 
+ early_initcall(init_trace_printk);
++#endif
 diff --git a/kernel/trace/trace_seq.c b/kernel/trace/trace_seq.c
 index e694c9f..6775a38 100644
 --- a/kernel/trace/trace_seq.c
@@ -157013,7 +157167,7 @@ index bf262e4..c5bc390 100644
  
  	if (S_ISREG(inode->i_mode))
 diff --git a/mm/usercopy.c b/mm/usercopy.c
-index 3c8da0a..3e4bdaf 100644
+index 3c8da0a..5c81035 100644
 --- a/mm/usercopy.c
 +++ b/mm/usercopy.c
 @@ -16,15 +16,9 @@
@@ -157091,7 +157245,25 @@ index 3c8da0a..3e4bdaf 100644
  }
  
  /* Returns true if any portion of [ptr,ptr+n) over laps with [low,high). */
-@@ -252,10 +252,15 @@ void __check_object_size(const void *ptr, unsigned long n, bool to_user)
+@@ -93,8 +93,17 @@ static bool overlaps(const void *ptr, unsigned long n, unsigned long low,
+ static inline const char *check_kernel_text_object(const void *ptr,
+ 						   unsigned long n)
+ {
++#if defined(CONFIG_X86_32) && defined(CONFIG_PAX_KERNEXEC)
++        unsigned long textlow = ktla_ktva((unsigned long)_stext);
++#ifdef CONFIG_MODULES
++        unsigned long texthigh = (unsigned long)MODULES_EXEC_VADDR;
++#else
++        unsigned long texthigh = ktla_ktva((unsigned long)_etext);
++#endif
++#else
+ 	unsigned long textlow = (unsigned long)_stext;
+ 	unsigned long texthigh = (unsigned long)_etext;
++#endif
+ 	unsigned long textlow_linear, texthigh_linear;
+ 
+ 	if (overlaps(ptr, n, textlow, texthigh))
+@@ -252,10 +261,15 @@ void __check_object_size(const void *ptr, unsigned long n, bool to_user)
  		goto report;
  
  	/* Check for bad stack object. */
@@ -157109,7 +157281,7 @@ index 3c8da0a..3e4bdaf 100644
  	case GOOD_FRAME:
  	case GOOD_STACK:
  		/*
-@@ -264,16 +269,12 @@ void __check_object_size(const void *ptr, unsigned long n, bool to_user)
+@@ -264,16 +278,12 @@ void __check_object_size(const void *ptr, unsigned long n, bool to_user)
  		 * process stack (when frame checking not available).
  		 */
  		return;
@@ -158908,7 +159080,7 @@ index b7de71f..808387d 100644
  
  	return err;
 diff --git a/net/core/dev.c b/net/core/dev.c
-index 44b3ba4..34822e6 100644
+index 9ce9d72..6221b92 100644
 --- a/net/core/dev.c
 +++ b/net/core/dev.c
 @@ -1768,7 +1768,7 @@ int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
@@ -159503,7 +159675,7 @@ index 3864b4b6..d2cbe83 100644
  }
  
 diff --git a/net/core/sock.c b/net/core/sock.c
-index fd7b41e..71dae11 100644
+index 10acacc..8b564eb 100644
 --- a/net/core/sock.c
 +++ b/net/core/sock.c
 @@ -411,13 +411,13 @@ int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
@@ -159540,7 +159712,7 @@ index fd7b41e..71dae11 100644
  		goto discard_and_relse;
  	}
  
-@@ -889,19 +889,6 @@ set_rcvbuf:
+@@ -890,19 +890,6 @@ set_rcvbuf:
  		}
  		break;
  
@@ -159560,7 +159732,7 @@ index fd7b41e..71dae11 100644
  	case SO_ATTACH_REUSEPORT_CBPF:
  		ret = -EINVAL;
  		if (optlen == sizeof(struct sock_fprog)) {
-@@ -915,6 +902,20 @@ set_rcvbuf:
+@@ -916,6 +903,20 @@ set_rcvbuf:
  		}
  		break;
  
@@ -159581,7 +159753,7 @@ index fd7b41e..71dae11 100644
  	case SO_ATTACH_REUSEPORT_EBPF:
  		ret = -EINVAL;
  		if (optlen == sizeof(u32)) {
-@@ -928,6 +929,8 @@ set_rcvbuf:
+@@ -929,6 +930,8 @@ set_rcvbuf:
  		}
  		break;
  
@@ -159590,7 +159762,7 @@ index fd7b41e..71dae11 100644
  	case SO_DETACH_FILTER:
  		ret = sk_detach_filter(sk);
  		break;
-@@ -1037,12 +1040,12 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
+@@ -1038,12 +1041,12 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
  		struct timeval tm;
  	} v;
  
@@ -159606,7 +159778,7 @@ index fd7b41e..71dae11 100644
  		return -EINVAL;
  
  	memset(&v, 0, sizeof(v));
-@@ -1180,11 +1183,11 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
+@@ -1181,11 +1184,11 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
  
  	case SO_PEERNAME:
  	{
@@ -159620,7 +159792,7 @@ index fd7b41e..71dae11 100644
  			return -EINVAL;
  		if (copy_to_user(optval, address, len))
  			return -EFAULT;
-@@ -1272,7 +1275,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
+@@ -1273,7 +1276,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
  
  	if (len > lv)
  		len = lv;
@@ -159629,7 +159801,7 @@ index fd7b41e..71dae11 100644
  		return -EFAULT;
  lenout:
  	if (put_user(len, optlen))
-@@ -1536,7 +1539,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
+@@ -1537,7 +1540,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
  		newsk->sk_dst_cache	= NULL;
  		newsk->sk_wmem_queued	= 0;
  		newsk->sk_forward_alloc = 0;
@@ -159638,8 +159810,8 @@ index fd7b41e..71dae11 100644
  		newsk->sk_send_head	= NULL;
  		newsk->sk_userlocks	= sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
  
-@@ -1565,7 +1568,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
- 		newsk->sk_err	   = 0;
+@@ -1567,7 +1570,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
+ 		newsk->sk_err_soft = 0;
  		newsk->sk_priority = 0;
  		newsk->sk_incoming_cpu = raw_smp_processor_id();
 -		atomic64_set(&newsk->sk_cookie, 0);
@@ -159647,7 +159819,7 @@ index fd7b41e..71dae11 100644
  
  		cgroup_sk_alloc(&newsk->sk_cgrp_data);
  
-@@ -2497,7 +2500,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
+@@ -2499,7 +2502,7 @@ void sock_init_data(struct socket *sock, struct sock *sk)
  	 */
  	smp_wmb();
  	atomic_set(&sk->sk_refcnt, 1);
@@ -159656,7 +159828,7 @@ index fd7b41e..71dae11 100644
  }
  EXPORT_SYMBOL(sock_init_data);
  
-@@ -2621,6 +2624,7 @@ void sock_enable_timestamp(struct sock *sk, int flag)
+@@ -2623,6 +2626,7 @@ void sock_enable_timestamp(struct sock *sk, int flag)
  int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
  		       int level, int type)
  {
@@ -159664,7 +159836,7 @@ index fd7b41e..71dae11 100644
  	struct sock_exterr_skb *serr;
  	struct sk_buff *skb;
  	int copied, err;
-@@ -2642,7 +2646,8 @@ int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
+@@ -2644,7 +2648,8 @@ int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
  	sock_recv_timestamp(msg, sk, skb);
  
  	serr = SKB_EXT_ERR(skb);
@@ -159674,7 +159846,7 @@ index fd7b41e..71dae11 100644
  
  	msg->msg_flags |= MSG_ERRQUEUE;
  	err = copied;
-@@ -3094,7 +3099,7 @@ static __net_exit void proto_exit_net(struct net *net)
+@@ -3096,7 +3101,7 @@ static __net_exit void proto_exit_net(struct net *net)
  }
  
  
@@ -160290,7 +160462,7 @@ index e9f5622..975108b 100644
  	return nh->nh_saddr;
  }
 diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
-index 38abe70..5e03195 100644
+index 48734ee..3308079 100644
 --- a/net/ipv4/icmp.c
 +++ b/net/ipv4/icmp.c
 @@ -1228,7 +1228,7 @@ fail:
@@ -160892,7 +161064,7 @@ index 438f50c..96538dd 100644
  	.exit = raw_exit_net,
  };
 diff --git a/net/ipv4/route.c b/net/ipv4/route.c
-index 62c3ed0..a3694bf 100644
+index 2f23ef1..85c2f0d 100644
 --- a/net/ipv4/route.c
 +++ b/net/ipv4/route.c
 @@ -232,7 +232,7 @@ static const struct seq_operations rt_cache_seq_ops = {
@@ -160940,7 +161112,7 @@ index 62c3ed0..a3694bf 100644
  static u32 *ip_tstamps __read_mostly;
  
  /* In order to protect privacy, we add a perturbation to identifiers
-@@ -2779,34 +2779,34 @@ static struct ctl_table ipv4_route_flush_table[] = {
+@@ -2781,34 +2781,34 @@ static struct ctl_table ipv4_route_flush_table[] = {
  		.maxlen		= sizeof(int),
  		.mode		= 0200,
  		.proc_handler	= ipv4_sysctl_rtcache_flush,
@@ -160983,7 +161155,7 @@ index 62c3ed0..a3694bf 100644
  err_dup:
  	return -ENOMEM;
  }
-@@ -2821,7 +2821,7 @@ static __net_exit void sysctl_route_net_exit(struct net *net)
+@@ -2823,7 +2823,7 @@ static __net_exit void sysctl_route_net_exit(struct net *net)
  	kfree(tbl);
  }
  
@@ -160992,7 +161164,7 @@ index 62c3ed0..a3694bf 100644
  	.init = sysctl_route_net_init,
  	.exit = sysctl_route_net_exit,
  };
-@@ -2829,14 +2829,14 @@ static __net_initdata struct pernet_operations sysctl_route_ops = {
+@@ -2831,14 +2831,14 @@ static __net_initdata struct pernet_operations sysctl_route_ops = {
  
  static __net_init int rt_genid_init(struct net *net)
  {
@@ -161010,7 +161182,7 @@ index 62c3ed0..a3694bf 100644
  	.init = rt_genid_init,
  };
  
-@@ -2860,7 +2860,7 @@ static void __net_exit ipv4_inetpeer_exit(struct net *net)
+@@ -2862,7 +2862,7 @@ static void __net_exit ipv4_inetpeer_exit(struct net *net)
  	kfree(bp);
  }
  
@@ -161019,7 +161191,7 @@ index 62c3ed0..a3694bf 100644
  	.init	=	ipv4_inetpeer_init,
  	.exit	=	ipv4_inetpeer_exit,
  };
-@@ -2874,11 +2874,7 @@ int __init ip_rt_init(void)
+@@ -2876,11 +2876,7 @@ int __init ip_rt_init(void)
  	int rc = 0;
  	int cpu;
  
@@ -161198,7 +161370,7 @@ index a756b87..7237f19 100644
  		write_pnet(&ireq->ireq_net, sock_net(sk_listener));
  		ireq->ireq_family = sk_listener->sk_family;
 diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
-index 7158d4f..a25f7f8 100644
+index 7b235fa..a8dbb72 100644
 --- a/net/ipv4/tcp_ipv4.c
 +++ b/net/ipv4/tcp_ipv4.c
 @@ -88,6 +88,10 @@ int sysctl_tcp_tw_reuse __read_mostly;
@@ -161222,7 +161394,7 @@ index 7158d4f..a25f7f8 100644
  	tcp_v4_send_reset(rsk, skb);
  discard:
  	kfree_skb(skb);
-@@ -1595,12 +1602,19 @@ int tcp_v4_rcv(struct sk_buff *skb)
+@@ -1610,12 +1617,19 @@ int tcp_v4_rcv(struct sk_buff *skb)
  lookup:
  	sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
  			       th->dest, &refcounted);
@@ -161245,7 +161417,7 @@ index 7158d4f..a25f7f8 100644
  
  	if (sk->sk_state == TCP_NEW_SYN_RECV) {
  		struct request_sock *req = inet_reqsk(sk);
-@@ -1690,6 +1704,10 @@ csum_error:
+@@ -1707,6 +1721,10 @@ csum_error:
  bad_packet:
  		__TCP_INC_STATS(net, TCP_MIB_INERRS);
  	} else {
@@ -161256,7 +161428,7 @@ index 7158d4f..a25f7f8 100644
  		tcp_v4_send_reset(NULL, skb);
  	}
  
-@@ -2428,7 +2446,7 @@ static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
+@@ -2445,7 +2463,7 @@ static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
  	inet_twsk_purge(&tcp_hashinfo, &tcp_death_row, AF_INET);
  }
  
@@ -161759,7 +161931,7 @@ index 37874e2..d5cd498 100644
 +		   atomic_read_unchecked(&sp->sk_drops));
  }
 diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c
-index bd59c34..8bd19fd2 100644
+index 7370ad2..7002ca8 100644
 --- a/net/ipv6/icmp.c
 +++ b/net/ipv6/icmp.c
 @@ -1073,7 +1073,7 @@ static struct ctl_table ipv6_icmp_table_template[] = {
@@ -162284,7 +162456,7 @@ index 69c50e7..ec875fa 100644
  	struct ctl_table *ipv6_icmp_table;
  	int err;
 diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
-index fc67822..e20c9c3 100644
+index af6a09e..3dafd52 100644
 --- a/net/ipv6/tcp_ipv6.c
 +++ b/net/ipv6/tcp_ipv6.c
 @@ -101,6 +101,10 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
@@ -162331,7 +162503,7 @@ index fc67822..e20c9c3 100644
  
  	if (sk->sk_state == TCP_NEW_SYN_RECV) {
  		struct request_sock *req = inet_reqsk(sk);
-@@ -1498,6 +1513,10 @@ csum_error:
+@@ -1500,6 +1515,10 @@ csum_error:
  bad_packet:
  		__TCP_INC_STATS(net, TCP_MIB_INERRS);
  	} else {
@@ -165720,10 +165892,10 @@ index 12d4519..367dae4 100644
  		NULL, sctp_cname, sctp_tname, sctp_oname, sctp_pname,
  	};
 diff --git a/net/sctp/socket.c b/net/sctp/socket.c
-index baccbf3..aed4ec2 100644
+index 7b0e059..96ff619 100644
 --- a/net/sctp/socket.c
 +++ b/net/sctp/socket.c
-@@ -2199,11 +2199,13 @@ static int sctp_setsockopt_events(struct sock *sk, char __user *optval,
+@@ -2202,11 +2202,13 @@ static int sctp_setsockopt_events(struct sock *sk, char __user *optval,
  {
  	struct sctp_association *asoc;
  	struct sctp_ulpevent *event;
@@ -165738,7 +165910,7 @@ index baccbf3..aed4ec2 100644
  
  	/* At the time when a user app subscribes to SCTP_SENDER_DRY_EVENT,
  	 * if there is no data to be sent or retransmit, the stack will
-@@ -4683,13 +4685,16 @@ static int sctp_getsockopt_disable_fragments(struct sock *sk, int len,
+@@ -4685,13 +4687,16 @@ static int sctp_getsockopt_disable_fragments(struct sock *sk, int len,
  static int sctp_getsockopt_events(struct sock *sk, int len, char __user *optval,
  				  int __user *optlen)
  {
@@ -165756,7 +165928,7 @@ index baccbf3..aed4ec2 100644
  		return -EFAULT;
  	return 0;
  }
-@@ -4707,6 +4712,8 @@ static int sctp_getsockopt_events(struct sock *sk, int len, char __user *optval,
+@@ -4709,6 +4714,8 @@ static int sctp_getsockopt_events(struct sock *sk, int len, char __user *optval,
   */
  static int sctp_getsockopt_autoclose(struct sock *sk, int len, char __user *optval, int __user *optlen)
  {
@@ -165765,7 +165937,7 @@ index baccbf3..aed4ec2 100644
  	/* Applicable to UDP-style socket only */
  	if (sctp_style(sk, TCP))
  		return -EOPNOTSUPP;
-@@ -4715,7 +4722,8 @@ static int sctp_getsockopt_autoclose(struct sock *sk, int len, char __user *optv
+@@ -4717,7 +4724,8 @@ static int sctp_getsockopt_autoclose(struct sock *sk, int len, char __user *optv
  	len = sizeof(int);
  	if (put_user(len, optlen))
  		return -EFAULT;
@@ -165775,7 +165947,7 @@ index baccbf3..aed4ec2 100644
  		return -EFAULT;
  	return 0;
  }
-@@ -5089,12 +5097,15 @@ static int sctp_getsockopt_delayed_ack(struct sock *sk, int len,
+@@ -5091,12 +5099,15 @@ static int sctp_getsockopt_delayed_ack(struct sock *sk, int len,
   */
  static int sctp_getsockopt_initmsg(struct sock *sk, int len, char __user *optval, int __user *optlen)
  {
@@ -165792,7 +165964,7 @@ index baccbf3..aed4ec2 100644
  		return -EFAULT;
  	return 0;
  }
-@@ -5135,6 +5146,8 @@ static int sctp_getsockopt_peer_addrs(struct sock *sk, int len,
+@@ -5137,6 +5148,8 @@ static int sctp_getsockopt_peer_addrs(struct sock *sk, int len,
  			      ->addr_to_user(sp, &temp);
  		if (space_left < addrlen)
  			return -ENOMEM;
@@ -165851,7 +166023,7 @@ index daf8554..4ab495b 100644
  
  	table = kmemdup(sctp_net_table, sizeof(sctp_net_table), GFP_KERNEL);
 diff --git a/net/socket.c b/net/socket.c
-index a1bd161..b1f7508 100644
+index 03bc2c2..d3f4699 100644
 --- a/net/socket.c
 +++ b/net/socket.c
 @@ -89,6 +89,7 @@
@@ -166061,7 +166233,7 @@ index a1bd161..b1f7508 100644
  				   ctl_len))
  			goto out_freectl;
  		msg_sys->msg_control = ctl_buf;
-@@ -2073,7 +2138,7 @@ static int ___sys_recvmsg(struct socket *sock, struct user_msghdr __user *msg,
+@@ -2075,7 +2140,7 @@ static int ___sys_recvmsg(struct socket *sock, struct user_msghdr __user *msg,
  	ssize_t err;
  
  	/* kernel mode address */
@@ -166070,7 +166242,7 @@ index a1bd161..b1f7508 100644
  
  	/* user mode address pointers */
  	struct sockaddr __user *uaddr;
-@@ -2719,7 +2784,7 @@ static int ethtool_ioctl(struct net *net, struct compat_ifreq __user *ifr32)
+@@ -2721,7 +2786,7 @@ static int ethtool_ioctl(struct net *net, struct compat_ifreq __user *ifr32)
  	ifr = compat_alloc_user_space(buf_size);
  	rxnfc = (void __user *)ifr + ALIGN(sizeof(struct ifreq), 8);
  
@@ -166079,7 +166251,7 @@ index a1bd161..b1f7508 100644
  		return -EFAULT;
  
  	if (put_user(convert_in ? rxnfc : compat_ptr(data),
-@@ -2830,7 +2895,7 @@ static int bond_ioctl(struct net *net, unsigned int cmd,
+@@ -2832,7 +2897,7 @@ static int bond_ioctl(struct net *net, unsigned int cmd,
  		old_fs = get_fs();
  		set_fs(KERNEL_DS);
  		err = dev_ioctl(net, cmd,
@@ -166088,7 +166260,7 @@ index a1bd161..b1f7508 100644
  		set_fs(old_fs);
  
  		return err;
-@@ -2923,7 +2988,7 @@ static int compat_sioc_ifmap(struct net *net, unsigned int cmd,
+@@ -2925,7 +2990,7 @@ static int compat_sioc_ifmap(struct net *net, unsigned int cmd,
  
  	old_fs = get_fs();
  	set_fs(KERNEL_DS);
@@ -166097,7 +166269,7 @@ index a1bd161..b1f7508 100644
  	set_fs(old_fs);
  
  	if (cmd == SIOCGIFMAP && !err) {
-@@ -3007,7 +3072,7 @@ static int routing_ioctl(struct net *net, struct socket *sock,
+@@ -3009,7 +3074,7 @@ static int routing_ioctl(struct net *net, struct socket *sock,
  		ret |= get_user(rtdev, &(ur4->rt_dev));
  		if (rtdev) {
  			ret |= copy_from_user(devname, compat_ptr(rtdev), 15);
@@ -166106,7 +166278,7 @@ index a1bd161..b1f7508 100644
  			devname[15] = 0;
  		} else
  			r4.rt_dev = NULL;
-@@ -3234,8 +3299,8 @@ int kernel_getsockopt(struct socket *sock, int level, int optname,
+@@ -3236,8 +3301,8 @@ int kernel_getsockopt(struct socket *sock, int level, int optname,
  	int __user *uoptlen;
  	int err;
  
@@ -166117,7 +166289,7 @@ index a1bd161..b1f7508 100644
  
  	set_fs(KERNEL_DS);
  	if (level == SOL_SOCKET)
-@@ -3255,7 +3320,7 @@ int kernel_setsockopt(struct socket *sock, int level, int optname,
+@@ -3257,7 +3322,7 @@ int kernel_setsockopt(struct socket *sock, int level, int optname,
  	char __user *uoptval;
  	int err;
  
@@ -211287,7 +211459,7 @@ index 0000000..5ea5f35
 +
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow.h b/scripts/gcc-plugins/size_overflow_plugin/size_overflow.h
 new file mode 100644
-index 0000000..8d14f68
+index 0000000..6ab0335
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow.h
 @@ -0,0 +1,345 @@
@@ -211351,7 +211523,7 @@ index 0000000..8d14f68
 +struct interesting_stmts;
 +typedef struct interesting_stmts * interesting_stmts_t;
 +
-+enum decl_type {
++enum based_decl {
 +	SO_FUNCTION, SO_VAR, SO_FIELD, SO_FUNCTION_POINTER, SO_AUX, SO_DISABLE, SO_NONE
 +};
 +
@@ -211364,7 +211536,7 @@ index 0000000..8d14f68
 +	unsigned int hash;
 +	unsigned int num;
 +	enum size_overflow_mark marked;
-+	enum decl_type decl_type;
++	enum based_decl based_decl;
 +	const char *orig_decl_str;
 +	unsigned int orig_num;
 +};
@@ -211454,7 +211626,7 @@ index 0000000..8d14f68
 +#endif
 +	const char *decl_name;
 +	const char *context;
-+	enum decl_type decl_type;
++	enum based_decl based_decl;
 +	unsigned int hash;
 +	unsigned int num;
 +	enum size_overflow_mark marked;
@@ -211486,7 +211658,7 @@ index 0000000..8d14f68
 +extern const struct size_overflow_hash *get_size_overflow_hash_entry_tree(struct fn_raw_data *raw_data, bool hash_table);
 +extern unsigned int find_arg_number_tree(const_tree arg, const_tree func);
 +extern unsigned int get_decl_hash(const_tree decl, const char *decl_name);
-+extern const char *get_decl_type_str(enum decl_type decl_type);
++extern const char *get_based_decl_str(enum based_decl based_decl);
 +extern void initialize_raw_data(struct fn_raw_data *raw_data);
 +
 +
@@ -211638,7 +211810,7 @@ index 0000000..8d14f68
 +#endif
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow_debug.c b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_debug.c
 new file mode 100644
-index 0000000..7101094
+index 0000000..663b6c9
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_debug.c
 @@ -0,0 +1,201 @@
@@ -211723,7 +211895,7 @@ index 0000000..7101094
 +#endif
 +
 +	fprintf(stderr, "print_next_interesting_function: ptr: %p, ", node);
-+	fprintf(stderr, "decl_name: %s, decl_type: %s, ", node->decl_name, get_decl_type_str(node->decl_type));
++	fprintf(stderr, "decl_name: %s, based_decl: %s, ", node->decl_name, get_based_decl_str(node->based_decl));
 +
 +	fprintf(stderr, "num: %u marked: %s context: %s children len: %u\n", node->num, print_so_mark_name(node->marked), node->context, children_len);
 +#if BUILDING_GCC_VERSION <= 4007
@@ -211841,14 +212013,14 @@ index 0000000..7101094
 +{
 +	fprintf(stderr, "decl_str: %s, context: %s, num: %u, hash: %u\ndecl:\n", data->decl_str ? data->decl_str : "NULL", data->context ? data->context : "NULL", data->num, data->hash);
 +	debug_tree(data->decl);
-+	fprintf(stderr, "marked: %s, decl_type: %s\norig_decl_str: %s, orig_num: %u\n", print_so_mark_name(data->marked), get_decl_type_str(data->decl_type), data->orig_decl_str? data->orig_decl_str : "NULL", data->orig_num);
++	fprintf(stderr, "marked: %s, based_decl: %s\norig_decl_str: %s, orig_num: %u\n", print_so_mark_name(data->marked), get_based_decl_str(data->based_decl), data->orig_decl_str? data->orig_decl_str : "NULL", data->orig_num);
 +}
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow_ipa.c b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_ipa.c
 new file mode 100644
-index 0000000..c384be8
+index 0000000..34b9974
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_ipa.c
-@@ -0,0 +1,1286 @@
+@@ -0,0 +1,1333 @@
 +/*
 + * Copyright 2011-2016 by Emese Revfy <re.emese@gmail.com>
 + * Licensed under the GPL v2
@@ -211875,8 +212047,6 @@ index 0000000..c384be8
 +
 +next_interesting_function_t global_next_interesting_function[GLOBAL_NIFN_LEN];
 +static bool global_changed;
-+#define PRINT_DATA_FLOW true
-+#define NO_PRINT_DATA_FLOW false
 +
 +static struct cgraph_node_hook_list *function_insertion_hook_holder;
 +static struct cgraph_2node_hook_list *node_duplication_hook_holder;
@@ -211985,7 +212155,7 @@ index 0000000..c384be8
 +	gcc_assert(raw_data->context);
 +	gcc_assert(raw_data->hash != NO_HASH);
 +	gcc_assert(raw_data->num != NONE_ARGNUM);
-+	gcc_assert(raw_data->decl_type != SO_NONE);
++	gcc_assert(raw_data->based_decl != SO_NONE);
 +
 +	new_node = (next_interesting_function_t)xmalloc(sizeof(*new_node));
 +	new_node->decl_name = xstrdup(raw_data->decl_str);
@@ -211998,7 +212168,7 @@ index 0000000..c384be8
 +	new_node->children = NULL;
 +	new_node->marked = raw_data->marked;
 +	new_node->orig_next_node = orig_next_node;
-+	new_node->decl_type = raw_data->decl_type;
++	new_node->based_decl = raw_data->based_decl;
 +
 +	return new_node;
 +}
@@ -212108,7 +212278,7 @@ index 0000000..c384be8
 +
 +	gcc_assert(clone_raw_data->decl != NULL_TREE);
 +	gcc_assert(clone_raw_data->num != NONE_ARGNUM);
-+	gcc_assert(clone_raw_data->decl_type != SO_NONE);
++	gcc_assert(clone_raw_data->based_decl != SO_NONE);
 +
 +	initialize_raw_data(&orig_raw_data);
 +	orig_raw_data.decl = get_orig_fndecl(clone_raw_data->decl);
@@ -212136,7 +212306,7 @@ index 0000000..c384be8
 +		return orig_next_node;
 +
 +	orig_raw_data.marked = clone_raw_data->marked;
-+	orig_raw_data.decl_type = clone_raw_data->decl_type;
++	orig_raw_data.based_decl = clone_raw_data->based_decl;
 +	orig_next_node = create_new_next_interesting_decl(&orig_raw_data, NULL);
 +	if (!orig_next_node)
 +		return NULL;
@@ -212220,7 +212390,7 @@ index 0000000..c384be8
 +
 +	initialize_raw_data(&raw_data);
 +	raw_data.decl = fndecl;
-+	raw_data.decl_type = SO_FUNCTION;
++	raw_data.based_decl = SO_FUNCTION;
 +	raw_data.decl_str = DECL_NAME_POINTER(fndecl);
 +	raw_data.marked = NO_SO_MARK;
 +
@@ -212323,7 +212493,7 @@ index 0000000..c384be8
 +	case INDIRECT_REF:
 +	case COMPONENT_REF:
 +		raw_data.decl = get_ref_field(node);
-+		raw_data.decl_type = SO_FIELD;
++		raw_data.based_decl = SO_FIELD;
 +		break;
 +	// TODO
 +	case BIT_FIELD_REF:
@@ -212349,7 +212519,7 @@ index 0000000..c384be8
 +	initialize_raw_data(&raw_data);
 +
 +	raw_data.decl = node;
-+	raw_data.decl_type = SO_VAR;
++	raw_data.based_decl = SO_VAR;
 +	create_and_append_new_next_interesting_field_var_decl(use_def_data, &raw_data);
 +}
 +
@@ -212529,7 +212699,7 @@ index 0000000..c384be8
 +	initialize_raw_data(&raw_data);
 +	raw_data.num = num;
 +	raw_data.marked = NO_SO_MARK;
-+	raw_data.decl_type = SO_FUNCTION;
++	raw_data.based_decl = SO_FUNCTION;
 +
 +	switch (gimple_code(stmt)) {
 +	case GIMPLE_ASM:
@@ -212726,7 +212896,7 @@ index 0000000..c384be8
 +		dst_raw_data.decl = NODE_DECL(dst);
 +		dst_raw_data.decl_str = cgraph_node_name(dst);
 +		dst_raw_data.marked = cur->marked;
-+		dst_raw_data.decl_type = cur->decl_type;
++		dst_raw_data.based_decl = cur->based_decl;
 +
 +		if (!made_by_compiler(dst_raw_data.decl))
 +			break;
@@ -212931,7 +213101,7 @@ index 0000000..c384be8
 +#endif
 +}
 +
-+static void walk_so_marked_fns(next_interesting_function_set *visited, next_interesting_function_t parent, bool debug)
++static void walk_so_marked_fns(next_interesting_function_set *visited, next_interesting_function_t parent)
 +{
 +	unsigned int i;
 +	next_interesting_function_t child;
@@ -212949,13 +213119,8 @@ index 0000000..c384be8
 +#endif
 +		set_yes_so_mark(child);
 +
-+		if (in_lto_p && debug == PRINT_DATA_FLOW) {
-+			fprintf(stderr, " PARENT: decl: %s-%u context: %s %p\n", parent->decl_name, parent->num, parent->context, parent);
-+			fprintf(stderr, " \tCHILD: decl: %s-%u context: %s %p\n", child->decl_name, child->num, child->context, child);
-+		}
-+
 +		if (!pointer_set_insert(visited, child))
-+			walk_so_marked_fns(visited, child, debug);
++			walk_so_marked_fns(visited, child);
 +	}
 +}
 +
@@ -212983,26 +213148,19 @@ index 0000000..c384be8
 +}
 +
 +// Set YES_SO_MARK on functions that will be emitted into the hash table
-+static void search_so_marked_fns(bool debug)
++static void search_so_marked_fns(void)
 +{
 +
 +	unsigned int i;
 +	next_interesting_function_set *visited;
-+	next_interesting_function_t cur_global;
 +
 +	visited = next_interesting_function_pointer_set_create();
 +	for (i = 0; i < GLOBAL_NIFN_LEN; i++) {
-+		for (cur_global = global_next_interesting_function[i]; cur_global; cur_global = cur_global->next) {
-+			if (cur_global->marked == NO_SO_MARK || pointer_set_insert(visited, cur_global))
-+				continue;
-+
-+			if  (in_lto_p && debug == PRINT_DATA_FLOW)
-+				fprintf(stderr, "Data flow: decl: %s-%u context: %s %p\n", cur_global->decl_name, cur_global->num, cur_global->context, cur_global);
++		next_interesting_function_t cur_global;
 +
-+			walk_so_marked_fns(visited, cur_global, debug);
-+
-+			if  (in_lto_p && debug == PRINT_DATA_FLOW)
-+				fprintf(stderr, "\n");
++		for (cur_global = global_next_interesting_function[i]; cur_global; cur_global = cur_global->next) {
++			if (cur_global->marked != NO_SO_MARK && !pointer_set_insert(visited, cur_global))
++				walk_so_marked_fns(visited, cur_global);
 +		}
 +	}
 +	pointer_set_destroy(visited);
@@ -213023,14 +213181,14 @@ index 0000000..c384be8
 +#else
 +	FOR_EACH_VEC_SAFE_ELT(parent->children, i, child) {
 +#endif
-+		switch (parent->decl_type) {
++		switch (parent->based_decl) {
 +		case SO_FIELD:
-+			child->decl_type = SO_FIELD;
-+			gcc_assert(child->decl_type != SO_FUNCTION_POINTER);
++			child->based_decl = SO_FIELD;
++			gcc_assert(child->based_decl != SO_FUNCTION_POINTER);
 +			break;
 +		case SO_FUNCTION_POINTER:
-+			child->decl_type = SO_FUNCTION_POINTER;
-+			gcc_assert(child->decl_type != SO_FIELD);
++			child->based_decl = SO_FUNCTION_POINTER;
++			gcc_assert(child->based_decl != SO_FIELD);
 +			break;
 +		case SO_FUNCTION:
 +		case SO_VAR:
@@ -213043,14 +213201,15 @@ index 0000000..c384be8
 +	}
 +}
 +
-+static void set_base_decl_type(void)
++static void set_based_decl(void)
 +{
 +	unsigned int i;
 +	next_interesting_function_set *visited;
-+	next_interesting_function_t cur;
 +
 +	visited = next_interesting_function_pointer_set_create();
 +	for (i = 0; i < GLOBAL_NIFN_LEN; i++) {
++		next_interesting_function_t cur;
++
 +		for (cur = global_next_interesting_function[i]; cur; cur = cur->next) {
 +			if (cur->marked == ASM_STMT_SO_MARK && !pointer_set_contains(visited, cur))
 +				walk_marked_functions(visited, cur);
@@ -213064,10 +213223,11 @@ index 0000000..c384be8
 +{
 +	unsigned int i;
 +	next_interesting_function_set *visited;
-+	next_interesting_function_t cur_global;
 +
 +	visited = next_interesting_function_pointer_set_create();
 +	for (i = 0; i < GLOBAL_NIFN_LEN; i++) {
++		next_interesting_function_t cur_global;
++
 +		for (cur_global = global_next_interesting_function[i]; cur_global; cur_global = cur_global->next) {
 +			if (cur_global->marked != NO_SO_MARK && !pointer_set_insert(visited, cur_global))
 +				print_missing_functions(visited, cur_global);
@@ -213078,31 +213238,90 @@ index 0000000..c384be8
 +
 +void __attribute__((weak)) check_global_variables(next_interesting_function_t cur_global __unused) {}
 +
-+// Print all missing interesting functions
-+static unsigned int size_overflow_execute(void)
++static void global_vars_and_fptrs(void)
 +{
 +	unsigned int i;
-+	next_interesting_function_t cur_global;
 +
-+	if (flag_lto && !in_lto_p)
-+		return 0;
++	if (!in_lto_p)
++		return;
 +
 +	// Collect vardecls and funtions reachable by function pointers
 +	for (i = 0; i < GLOBAL_NIFN_LEN; i++) {
++		next_interesting_function_t cur_global;
++
 +		for (cur_global = global_next_interesting_function[i]; cur_global; cur_global = cur_global->next) {
 +			check_global_variables(cur_global);
 +			search_missing_fptr_arg(cur_global);
 +		}
 +	}
++}
 +
-+	search_so_marked_fns(PRINT_DATA_FLOW);
-+	while (global_changed) {
-+		global_changed = false;
-+		search_so_marked_fns(NO_PRINT_DATA_FLOW);
++static void print_parent_child(next_interesting_function_set *visited, next_interesting_function_t parent)
++{
++	unsigned int i;
++	next_interesting_function_t child;
++
++#if BUILDING_GCC_VERSION <= 4007
++	if (VEC_empty(next_interesting_function_t, parent->children))
++		return;
++	FOR_EACH_VEC_ELT(next_interesting_function_t, parent->children, i, child) {
++#else
++	FOR_EACH_VEC_SAFE_ELT(parent->children, i, child) {
++#endif
++		fprintf(stderr, " PARENT: decl: %s-%u context: %s %p\n", parent->decl_name, parent->num, parent->context, parent);
++		fprintf(stderr, " \tCHILD: decl: %s-%u context: %s %p\n", child->decl_name, child->num, child->context, child);
++
++		if (!pointer_set_insert(visited, child))
++			print_parent_child(visited, child);
++	}
++}
++
++static void print_data_flow(void)
++{
++	unsigned int i;
++	next_interesting_function_set *visited;
++
++	if (!in_lto_p)
++		return;
++
++	visited = next_interesting_function_pointer_set_create();
++	for (i = 0; i < GLOBAL_NIFN_LEN; i++) {
++		next_interesting_function_t cur_global;
++
++		for (cur_global = global_next_interesting_function[i]; cur_global; cur_global = cur_global->next) {
++			if (cur_global->marked == NO_SO_MARK || pointer_set_insert(visited, cur_global))
++				continue;
++
++			fprintf(stderr, "Data flow: decl: %s-%u context: %s %p\n", cur_global->decl_name, cur_global->num, cur_global->context, cur_global);
++
++			print_parent_child(visited, cur_global);
++
++			fprintf(stderr, "\n");
++		}
 +	}
++	pointer_set_destroy(visited);
++}
++
++static void set_so_fns(void)
++{
++	do {
++		global_changed = false;
++		search_so_marked_fns();
++	} while (global_changed);
++
++	print_data_flow();
++}
++
++// Print all missing interesting functions
++static unsigned int size_overflow_execute(void)
++{
++	if (flag_lto && !in_lto_p)
++		return 0;
 +
-+	set_base_decl_type();
++	global_vars_and_fptrs();
 +
++	set_so_fns();
++	set_based_decl();
 +	print_so_marked_fns();
 +
 +	if (in_lto_p) {
@@ -213648,7 +213867,7 @@ index 0000000..b5291e1
 +
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin.c b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin.c
 new file mode 100644
-index 0000000..e3b24b1
+index 0000000..df0c102
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin.c
 @@ -0,0 +1,313 @@
@@ -213685,7 +213904,7 @@ index 0000000..e3b24b1
 +bool check_fields, check_fns, check_fnptrs, check_vars;
 +
 +static struct plugin_info size_overflow_plugin_info = {
-+	.version	= "20161025",
++	.version	= "20161113",
 +	.help		= "no-size-overflow\tturn off size overflow checking\n",
 +};
 +
@@ -213967,7 +214186,7 @@ index 0000000..e3b24b1
 +}
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin_hash.c b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin_hash.c
 new file mode 100644
-index 0000000..caf4bf1
+index 0000000..67cacb9
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_plugin_hash.c
 @@ -0,0 +1,469 @@
@@ -214006,7 +214225,7 @@ index 0000000..caf4bf1
 +	raw_data->context = NULL;
 +	raw_data->hash = NO_HASH;
 +	raw_data->num = NONE_ARGNUM;
-+	raw_data->decl_type = SO_NONE;
++	raw_data->based_decl = SO_NONE;
 +	raw_data->orig_decl_str = NULL;
 +	raw_data->orig_num = NONE_ARGNUM;
 +}
@@ -214098,7 +214317,7 @@ index 0000000..caf4bf1
 +	decl_hash_data->hash = type ^ fn ^ codes;
 +}
 +
-+static void set_decl_type_codes(const_tree type, struct decl_hash *decl_hash_data)
++static void set_based_decl_codes(const_tree type, struct decl_hash *decl_hash_data)
 +{
 +	gcc_assert(type != NULL_TREE);
 +	gcc_assert(TREE_CODE_CLASS(TREE_CODE(type)) == tcc_type);
@@ -214119,7 +214338,7 @@ index 0000000..caf4bf1
 +	if (DECL_P(node)) {
 +		result = DECL_RESULT(node);
 +		if (result != NULL_TREE)
-+			return set_decl_type_codes(TREE_TYPE(result), decl_hash_data);
++			return set_based_decl_codes(TREE_TYPE(result), decl_hash_data);
 +		return set_result_codes(TREE_TYPE(node), decl_hash_data);
 +	}
 +
@@ -214128,7 +214347,7 @@ index 0000000..caf4bf1
 +	if (TREE_CODE(node) == FUNCTION_TYPE)
 +		return set_result_codes(TREE_TYPE(node), decl_hash_data);
 +
-+	return set_decl_type_codes(node, decl_hash_data);
++	return set_based_decl_codes(node, decl_hash_data);
 +}
 +
 +static void set_decl_codes(struct decl_hash *decl_hash_data)
@@ -214137,7 +214356,7 @@ index 0000000..caf4bf1
 +	enum tree_code code;
 +
 +	if (TREE_CODE(decl_hash_data->decl) == VAR_DECL || TREE_CODE(decl_hash_data->decl) == FIELD_DECL) {
-+		set_decl_type_codes(TREE_TYPE(decl_hash_data->decl), decl_hash_data);
++		set_based_decl_codes(TREE_TYPE(decl_hash_data->decl), decl_hash_data);
 +		return;
 +	}
 +
@@ -214151,7 +214370,7 @@ index 0000000..caf4bf1
 +		set_result_codes(decl_hash_data->decl, decl_hash_data);
 +
 +	for (arg = TYPE_ARG_TYPES(type); arg != NULL_TREE && decl_hash_data->tree_codes_len < CODES_LIMIT; arg = TREE_CHAIN(arg))
-+		set_decl_type_codes(TREE_VALUE(arg), decl_hash_data);
++		set_based_decl_codes(TREE_VALUE(arg), decl_hash_data);
 +}
 +
 +static const struct size_overflow_hash *get_proper_hash_chain(const struct size_overflow_hash *entry, struct fn_raw_data *raw_data)
@@ -214233,7 +214452,7 @@ index 0000000..caf4bf1
 +	entry = disable_hash[raw_data->hash];
 +	entry_node = get_proper_hash_chain(entry, raw_data);
 +	if (entry_node && entry_node->param & (1U << raw_data->orig_num)) {
-+		raw_data->decl_type = SO_DISABLE;
++		raw_data->based_decl = SO_DISABLE;
 +		return entry_node;
 +	}
 +
@@ -214259,41 +214478,41 @@ index 0000000..caf4bf1
 +{
 +	const struct size_overflow_hash *entry;
 +
-+	gcc_assert(raw_data->decl_type == SO_NONE);
++	gcc_assert(raw_data->based_decl == SO_NONE);
 +
 +	gcc_assert(raw_data->hash != NO_HASH);
 +
 +	entry = get_proper_node(fns_hash[raw_data->hash], raw_data);
 +	if (entry) {
-+		raw_data->decl_type = SO_FUNCTION;
++		raw_data->based_decl = SO_FUNCTION;
 +		return check_fns ? entry : NULL;
 +	}
 +
 +	entry = get_proper_node(fields_hash[raw_data->hash], raw_data);
 +	if (entry) {
-+		raw_data->decl_type = SO_FIELD;
++		raw_data->based_decl = SO_FIELD;
 +		return check_fields ? entry : NULL;
 +	}
 +
 +	entry = get_proper_node(vars_hash[raw_data->hash], raw_data);
 +	if (entry) {
-+		raw_data->decl_type = SO_VAR;
++		raw_data->based_decl = SO_VAR;
 +		return check_vars ? entry : NULL;
 +	}
 +
 +	entry = get_proper_node(fptrs_hash[raw_data->hash], raw_data);
 +	if (entry) {
-+		raw_data->decl_type = SO_FUNCTION_POINTER;
++		raw_data->based_decl = SO_FUNCTION_POINTER;
 +		return check_fnptrs ? entry : NULL;
 +	}
 +
 +	entry = get_proper_node(aux_hash[raw_data->hash], raw_data);
 +	if (entry) {
-+		raw_data->decl_type = SO_AUX;
++		raw_data->based_decl = SO_AUX;
 +		return entry;
 +	}
 +
-+	raw_data->decl_type = SO_NONE;
++	raw_data->based_decl = SO_NONE;
 +	return NULL;
 +}
 +
@@ -214362,9 +214581,9 @@ index 0000000..caf4bf1
 +	return NULL_TREE;
 +}
 +
-+const char *get_decl_type_str(enum decl_type decl_type)
++const char *get_based_decl_str(enum based_decl based_decl)
 +{
-+	switch (decl_type) {
++	switch (based_decl) {
 +	case SO_FUNCTION:
 +		return "fns";
 +	case SO_FIELD:
@@ -214390,13 +214609,13 @@ index 0000000..caf4bf1
 +void print_missing_function(next_interesting_function_t node)
 +{
 +	const struct size_overflow_hash *entry;
-+	const char *decl_type;
++	const char *based_decl;
 +	struct fn_raw_data raw_data;
 +
 +	if (node->marked == ASM_STMT_SO_MARK)
 +		return;
 +
-+	switch (node->decl_type) {
++	switch (node->based_decl) {
 +	case SO_FUNCTION:
 +		if (!check_fns)
 +			return;
@@ -214435,14 +214654,14 @@ index 0000000..caf4bf1
 +	if (entry)
 +		return;
 +
-+	decl_type = get_decl_type_str(node->decl_type);
++	based_decl = get_based_decl_str(node->based_decl);
 +	// inform() would be too slow
-+	fprintf(stderr, "Function %s is missing from the e_%s hash table +%s+%s+%u+%u+\n", raw_data.orig_decl_str, decl_type, raw_data.orig_decl_str, raw_data.context, raw_data.orig_num, raw_data.hash);
++	fprintf(stderr, "Function %s is missing from the e_%s hash table +%s+%s+%u+%u+\n", raw_data.orig_decl_str, based_decl, raw_data.orig_decl_str, raw_data.context, raw_data.orig_num, raw_data.hash);
 +}
 +
 diff --git a/scripts/gcc-plugins/size_overflow_plugin/size_overflow_transform.c b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_transform.c
 new file mode 100644
-index 0000000..23fa5f1
+index 0000000..620a44a
 --- /dev/null
 +++ b/scripts/gcc-plugins/size_overflow_plugin/size_overflow_transform.c
 @@ -0,0 +1,772 @@
@@ -214833,12 +215052,12 @@ index 0000000..23fa5f1
 +	so_hash = get_size_overflow_hash_entry_tree(&raw_data, SIZE_OVERFLOW);
 +	if (so_hash)
 +		return get_and_create_next_node_from_global_next_nodes(&raw_data, NULL);
-+	if (!so_hash && raw_data.decl_type != SO_NONE)
++	if (!so_hash && raw_data.based_decl != SO_NONE)
 +		return NULL;
 +
 +	next_node = get_global_next_interesting_function_entry_with_hash(&raw_data);
 +	if (next_node && next_node->marked != NO_SO_MARK) {
-+		switch (next_node->decl_type) {
++		switch (next_node->based_decl) {
 +		case SO_FUNCTION:
 +			if (check_fns)
 +				return next_node;
@@ -215049,7 +215268,7 @@ index 0000000..23fa5f1
 +	raw_data.hash = 0;
 +	raw_data.num = 0;
 +	raw_data.marked = ASM_STMT_SO_MARK;
-+	raw_data.decl_type = SO_FUNCTION;
++	raw_data.based_decl = SO_FUNCTION;
 +
 +	next_node = get_global_next_interesting_function_entry(&raw_data);
 +	if (next_node)
diff --git a/4.8.9/4425_grsec_remove_EI_PAX.patch b/4.8.10/4425_grsec_remove_EI_PAX.patch
index 594598a..594598a 100644
--- a/4.8.9/4425_grsec_remove_EI_PAX.patch
+++ b/4.8.10/4425_grsec_remove_EI_PAX.patch
diff --git a/4.8.9/4427_force_XATTR_PAX_tmpfs.patch b/4.8.10/4427_force_XATTR_PAX_tmpfs.patch
index 2562d2f..2562d2f 100644
--- a/4.8.9/4427_force_XATTR_PAX_tmpfs.patch
+++ b/4.8.10/4427_force_XATTR_PAX_tmpfs.patch
diff --git a/4.8.9/4430_grsec-remove-localversion-grsec.patch b/4.8.10/4430_grsec-remove-localversion-grsec.patch
index 31cf878..31cf878 100644
--- a/4.8.9/4430_grsec-remove-localversion-grsec.patch
+++ b/4.8.10/4430_grsec-remove-localversion-grsec.patch
diff --git a/4.8.9/4435_grsec-mute-warnings.patch b/4.8.10/4435_grsec-mute-warnings.patch
index 8929222..8929222 100644
--- a/4.8.9/4435_grsec-mute-warnings.patch
+++ b/4.8.10/4435_grsec-mute-warnings.patch
diff --git a/4.8.9/4440_grsec-remove-protected-paths.patch b/4.8.10/4440_grsec-remove-protected-paths.patch
index 741546d..741546d 100644
--- a/4.8.9/4440_grsec-remove-protected-paths.patch
+++ b/4.8.10/4440_grsec-remove-protected-paths.patch
diff --git a/4.8.9/4450_grsec-kconfig-default-gids.patch b/4.8.10/4450_grsec-kconfig-default-gids.patch
index 6fd0511..6fd0511 100644
--- a/4.8.9/4450_grsec-kconfig-default-gids.patch
+++ b/4.8.10/4450_grsec-kconfig-default-gids.patch
diff --git a/4.8.9/4465_selinux-avc_audit-log-curr_ip.patch b/4.8.10/4465_selinux-avc_audit-log-curr_ip.patch
index 7248385..7248385 100644
--- a/4.8.9/4465_selinux-avc_audit-log-curr_ip.patch
+++ b/4.8.10/4465_selinux-avc_audit-log-curr_ip.patch
diff --git a/4.8.9/4470_disable-compat_vdso.patch b/4.8.10/4470_disable-compat_vdso.patch
index 1e4b84a..1e4b84a 100644
--- a/4.8.9/4470_disable-compat_vdso.patch
+++ b/4.8.10/4470_disable-compat_vdso.patch
diff --git a/4.8.9/4475_emutramp_default_on.patch b/4.8.10/4475_emutramp_default_on.patch
index 7b468ee..7b468ee 100644
--- a/4.8.9/4475_emutramp_default_on.patch
+++ b/4.8.10/4475_emutramp_default_on.patch