diff options
Diffstat (limited to '4.8.11/1009_linux-4.8.10.patch')
| -rw-r--r-- | 4.8.11/1009_linux-4.8.10.patch | 4759 |
1 files changed, 0 insertions, 4759 deletions
diff --git a/4.8.11/1009_linux-4.8.10.patch b/4.8.11/1009_linux-4.8.10.patch deleted file mode 100644 index 1e751e5..0000000 --- a/4.8.11/1009_linux-4.8.10.patch +++ /dev/null @@ -1,4759 +0,0 @@ -diff --git a/Makefile b/Makefile -index c1519ab..7cf2b49 100644 ---- a/Makefile -+++ b/Makefile -@@ -1,6 +1,6 @@ - VERSION = 4 - PATCHLEVEL = 8 --SUBLEVEL = 9 -+SUBLEVEL = 10 - EXTRAVERSION = - NAME = Psychotic Stoned Sheep - -diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h -index 37a315d..a6847fc 100644 ---- a/arch/sparc/include/asm/uaccess_64.h -+++ b/arch/sparc/include/asm/uaccess_64.h -@@ -98,7 +98,6 @@ struct exception_table_entry { - unsigned int insn, fixup; - }; - --void __ret_efault(void); - void __retl_efault(void); - - /* Uh, these should become the main single-value transfer routines.. -@@ -205,55 +204,34 @@ int __get_user_bad(void); - unsigned long __must_check ___copy_from_user(void *to, - const void __user *from, - unsigned long size); --unsigned long copy_from_user_fixup(void *to, const void __user *from, -- unsigned long size); - static inline unsigned long __must_check - copy_from_user(void *to, const void __user *from, unsigned long size) - { -- unsigned long ret; -- - check_object_size(to, size, false); - -- ret = ___copy_from_user(to, from, size); -- if (unlikely(ret)) -- ret = copy_from_user_fixup(to, from, size); -- -- return ret; -+ return ___copy_from_user(to, from, size); - } - #define __copy_from_user copy_from_user - - unsigned long __must_check ___copy_to_user(void __user *to, - const void *from, - unsigned long size); --unsigned long copy_to_user_fixup(void __user *to, const void *from, -- unsigned long size); - static inline unsigned long __must_check - copy_to_user(void __user *to, const void *from, unsigned long size) - { -- unsigned long ret; -- - check_object_size(from, size, true); - -- ret = ___copy_to_user(to, from, size); -- if (unlikely(ret)) -- ret = copy_to_user_fixup(to, from, size); -- return ret; -+ return ___copy_to_user(to, from, size); - } - #define __copy_to_user copy_to_user - - unsigned long __must_check ___copy_in_user(void __user *to, - const void __user *from, - unsigned long size); --unsigned long copy_in_user_fixup(void __user *to, void __user *from, -- unsigned long size); - static inline unsigned long __must_check - copy_in_user(void __user *to, void __user *from, unsigned long size) - { -- unsigned long ret = ___copy_in_user(to, from, size); -- -- if (unlikely(ret)) -- ret = copy_in_user_fixup(to, from, size); -- return ret; -+ return ___copy_in_user(to, from, size); - } - #define __copy_in_user copy_in_user - -diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S -index a076b42..5f1f3ae 100644 ---- a/arch/sparc/kernel/head_64.S -+++ b/arch/sparc/kernel/head_64.S -@@ -922,47 +922,11 @@ prom_tba: .xword 0 - tlb_type: .word 0 /* Must NOT end up in BSS */ - .section ".fixup",#alloc,#execinstr - -- .globl __ret_efault, __retl_efault, __ret_one, __retl_one --ENTRY(__ret_efault) -- ret -- restore %g0, -EFAULT, %o0 --ENDPROC(__ret_efault) -- - ENTRY(__retl_efault) - retl - mov -EFAULT, %o0 - ENDPROC(__retl_efault) - --ENTRY(__retl_one) -- retl -- mov 1, %o0 --ENDPROC(__retl_one) -- --ENTRY(__retl_one_fp) -- VISExitHalf -- retl -- mov 1, %o0 --ENDPROC(__retl_one_fp) -- --ENTRY(__ret_one_asi) -- wr %g0, ASI_AIUS, %asi -- ret -- restore %g0, 1, %o0 --ENDPROC(__ret_one_asi) -- --ENTRY(__retl_one_asi) -- wr %g0, ASI_AIUS, %asi -- retl -- mov 1, %o0 --ENDPROC(__retl_one_asi) -- --ENTRY(__retl_one_asi_fp) -- wr %g0, ASI_AIUS, %asi -- VISExitHalf -- retl -- mov 1, %o0 --ENDPROC(__retl_one_asi_fp) -- - ENTRY(__retl_o1) - retl - mov %o1, %o0 -diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c -index 59bbeff..07933b9 100644 ---- a/arch/sparc/kernel/jump_label.c -+++ b/arch/sparc/kernel/jump_label.c -@@ -13,19 +13,30 @@ - void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) - { -- u32 val; - u32 *insn = (u32 *) (unsigned long) entry->code; -+ u32 val; - - if (type == JUMP_LABEL_JMP) { - s32 off = (s32)entry->target - (s32)entry->code; -+ bool use_v9_branch = false; -+ -+ BUG_ON(off & 3); - - #ifdef CONFIG_SPARC64 -- /* ba,pt %xcc, . + (off << 2) */ -- val = 0x10680000 | ((u32) off >> 2); --#else -- /* ba . + (off << 2) */ -- val = 0x10800000 | ((u32) off >> 2); -+ if (off <= 0xfffff && off >= -0x100000) -+ use_v9_branch = true; - #endif -+ if (use_v9_branch) { -+ /* WDISP19 - target is . + immed << 2 */ -+ /* ba,pt %xcc, . + off */ -+ val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); -+ } else { -+ /* WDISP22 - target is . + immed << 2 */ -+ BUG_ON(off > 0x7fffff); -+ BUG_ON(off < -0x800000); -+ /* ba . + off */ -+ val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); -+ } - } else { - val = 0x01000000; - } -diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c -index 9e034f2..20ffb05 100644 ---- a/arch/sparc/kernel/sparc_ksyms_64.c -+++ b/arch/sparc/kernel/sparc_ksyms_64.c -@@ -27,7 +27,6 @@ EXPORT_SYMBOL(__flushw_user); - EXPORT_SYMBOL_GPL(real_hard_smp_processor_id); - - /* from head_64.S */ --EXPORT_SYMBOL(__ret_efault); - EXPORT_SYMBOL(tlb_type); - EXPORT_SYMBOL(sun4v_chip_type); - EXPORT_SYMBOL(prom_root_node); -diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S -index b7d0bd6..69a439f 100644 ---- a/arch/sparc/lib/GENcopy_from_user.S -+++ b/arch/sparc/lib/GENcopy_from_user.S -@@ -3,11 +3,11 @@ - * Copyright (C) 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_LD(x) \ -+#define EX_LD(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S -index 780550e..9947427 100644 ---- a/arch/sparc/lib/GENcopy_to_user.S -+++ b/arch/sparc/lib/GENcopy_to_user.S -@@ -3,11 +3,11 @@ - * Copyright (C) 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S -index 89358ee..059ea24 100644 ---- a/arch/sparc/lib/GENmemcpy.S -+++ b/arch/sparc/lib/GENmemcpy.S -@@ -4,21 +4,18 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #define GLOBAL_SPARE %g7 - #else - #define GLOBAL_SPARE %g5 - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x --#endif -- --#ifndef EX_RETVAL --#define EX_RETVAL(x) x -+#define EX_ST(x,y) x - #endif - - #ifndef LOAD -@@ -45,6 +42,29 @@ - .register %g3,#scratch - - .text -+ -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+ENTRY(GEN_retl_o4_1) -+ add %o4, %o2, %o4 -+ retl -+ add %o4, 1, %o0 -+ENDPROC(GEN_retl_o4_1) -+ENTRY(GEN_retl_g1_8) -+ add %g1, %o2, %g1 -+ retl -+ add %g1, 8, %o0 -+ENDPROC(GEN_retl_g1_8) -+ENTRY(GEN_retl_o2_4) -+ retl -+ add %o2, 4, %o0 -+ENDPROC(GEN_retl_o2_4) -+ENTRY(GEN_retl_o2_1) -+ retl -+ add %o2, 1, %o0 -+ENDPROC(GEN_retl_o2_1) -+#endif -+ - .align 64 - - .globl FUNC_NAME -@@ -73,8 +93,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %g0, %o4, %o4 - sub %o2, %o4, %o2 - 1: subcc %o4, 1, %o4 -- EX_LD(LOAD(ldub, %o1, %g1)) -- EX_ST(STORE(stb, %g1, %o0)) -+ EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) -+ EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) - add %o1, 1, %o1 - bne,pt %XCC, 1b - add %o0, 1, %o0 -@@ -82,8 +102,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andn %o2, 0x7, %g1 - sub %o2, %g1, %o2 - 1: subcc %g1, 0x8, %g1 -- EX_LD(LOAD(ldx, %o1, %g2)) -- EX_ST(STORE(stx, %g2, %o0)) -+ EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) -+ EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) - add %o1, 0x8, %o1 - bne,pt %XCC, 1b - add %o0, 0x8, %o0 -@@ -100,8 +120,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 1: - subcc %o2, 4, %o2 -- EX_LD(LOAD(lduw, %o1, %g1)) -- EX_ST(STORE(stw, %g1, %o1 + %o3)) -+ EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) -+ EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -@@ -111,8 +131,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - .align 32 - 90: - subcc %o2, 1, %o2 -- EX_LD(LOAD(ldub, %o1, %g1)) -- EX_ST(STORE(stb, %g1, %o1 + %o3)) -+ EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) -+ EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl -diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile -index 3269b02..4f2384a 100644 ---- a/arch/sparc/lib/Makefile -+++ b/arch/sparc/lib/Makefile -@@ -38,7 +38,7 @@ lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o - lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o - lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o - --lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o -+lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o - lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o - - obj-$(CONFIG_SPARC64) += iomap.o -diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S -index d5242b8..b79a699 100644 ---- a/arch/sparc/lib/NG2copy_from_user.S -+++ b/arch/sparc/lib/NG2copy_from_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_LD(x) \ -+#define EX_LD(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_LD_FP(x) \ -+#define EX_LD_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S -index 4e962d9..dcec55f 100644 ---- a/arch/sparc/lib/NG2copy_to_user.S -+++ b/arch/sparc/lib/NG2copy_to_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_ST_FP(x) \ -+#define EX_ST_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S -index d5f585d..c629dbd 100644 ---- a/arch/sparc/lib/NG2memcpy.S -+++ b/arch/sparc/lib/NG2memcpy.S -@@ -4,6 +4,7 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #include <asm/visasm.h> - #include <asm/asi.h> - #define GLOBAL_SPARE %g7 -@@ -32,21 +33,17 @@ - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - #ifndef EX_LD_FP --#define EX_LD_FP(x) x -+#define EX_LD_FP(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x -+#define EX_ST(x,y) x - #endif - #ifndef EX_ST_FP --#define EX_ST_FP(x) x --#endif -- --#ifndef EX_RETVAL --#define EX_RETVAL(x) x -+#define EX_ST_FP(x,y) x - #endif - - #ifndef LOAD -@@ -140,45 +137,110 @@ - fsrc2 %x6, %f12; \ - fsrc2 %x7, %f14; - #define FREG_LOAD_1(base, x0) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)) -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) - #define FREG_LOAD_2(base, x0, x1) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); - #define FREG_LOAD_3(base, x0, x1, x2) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ -- EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); - #define FREG_LOAD_4(base, x0, x1, x2, x3) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ -- EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ -- EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); - #define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ -- EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ -- EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ -- EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); - #define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ -- EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ -- EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ -- EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ -- EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); - #define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ -- EX_LD_FP(LOAD(ldd, base + 0x00, %x0)); \ -- EX_LD_FP(LOAD(ldd, base + 0x08, %x1)); \ -- EX_LD_FP(LOAD(ldd, base + 0x10, %x2)); \ -- EX_LD_FP(LOAD(ldd, base + 0x18, %x3)); \ -- EX_LD_FP(LOAD(ldd, base + 0x20, %x4)); \ -- EX_LD_FP(LOAD(ldd, base + 0x28, %x5)); \ -- EX_LD_FP(LOAD(ldd, base + 0x30, %x6)); -+ EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ -+ EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); - - .register %g2,#scratch - .register %g3,#scratch - - .text -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+__restore_fp: -+ VISExitHalf -+__restore_asi: -+ retl -+ wr %g0, ASI_AIUS, %asi -+ENTRY(NG2_retl_o2) -+ ba,pt %xcc, __restore_asi -+ mov %o2, %o0 -+ENDPROC(NG2_retl_o2) -+ENTRY(NG2_retl_o2_plus_1) -+ ba,pt %xcc, __restore_asi -+ add %o2, 1, %o0 -+ENDPROC(NG2_retl_o2_plus_1) -+ENTRY(NG2_retl_o2_plus_4) -+ ba,pt %xcc, __restore_asi -+ add %o2, 4, %o0 -+ENDPROC(NG2_retl_o2_plus_4) -+ENTRY(NG2_retl_o2_plus_8) -+ ba,pt %xcc, __restore_asi -+ add %o2, 8, %o0 -+ENDPROC(NG2_retl_o2_plus_8) -+ENTRY(NG2_retl_o2_plus_o4_plus_1) -+ add %o4, 1, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG2_retl_o2_plus_o4_plus_1) -+ENTRY(NG2_retl_o2_plus_o4_plus_8) -+ add %o4, 8, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG2_retl_o2_plus_o4_plus_8) -+ENTRY(NG2_retl_o2_plus_o4_plus_16) -+ add %o4, 16, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG2_retl_o2_plus_o4_plus_16) -+ENTRY(NG2_retl_o2_plus_g1_fp) -+ ba,pt %xcc, __restore_fp -+ add %o2, %g1, %o0 -+ENDPROC(NG2_retl_o2_plus_g1_fp) -+ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) -+ add %g1, 64, %g1 -+ ba,pt %xcc, __restore_fp -+ add %o2, %g1, %o0 -+ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) -+ENTRY(NG2_retl_o2_plus_g1_plus_1) -+ add %g1, 1, %g1 -+ ba,pt %xcc, __restore_asi -+ add %o2, %g1, %o0 -+ENDPROC(NG2_retl_o2_plus_g1_plus_1) -+ENTRY(NG2_retl_o2_and_7_plus_o4) -+ and %o2, 7, %o2 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG2_retl_o2_and_7_plus_o4) -+ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) -+ and %o2, 7, %o2 -+ add %o4, 8, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) -+#endif -+ - .align 64 - - .globl FUNC_NAME -@@ -230,8 +292,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %g0, %o4, %o4 ! bytes to align dst - sub %o2, %o4, %o2 - 1: subcc %o4, 1, %o4 -- EX_LD(LOAD(ldub, %o1, %g1)) -- EX_ST(STORE(stb, %g1, %o0)) -+ EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) -+ EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) - add %o1, 1, %o1 - bne,pt %XCC, 1b - add %o0, 1, %o0 -@@ -281,11 +343,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - nop - /* fall through for 0 < low bits < 8 */ - 110: sub %o4, 64, %g2 -- EX_LD_FP(LOAD_BLK(%g2, %f0)) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+ EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -296,10 +358,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 120: sub %o4, 56, %g2 - FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -310,10 +372,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 130: sub %o4, 48, %g2 - FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_6(f20, f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -324,10 +386,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 140: sub %o4, 40, %g2 - FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_5(f22, f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -338,10 +400,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 150: sub %o4, 32, %g2 - FREG_LOAD_4(%g2, f0, f2, f4, f6) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_4(f24, f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -352,10 +414,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 160: sub %o4, 24, %g2 - FREG_LOAD_3(%g2, f0, f2, f4) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_3(f26, f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -366,10 +428,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 170: sub %o4, 16, %g2 - FREG_LOAD_2(%g2, f0, f2) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_2(f28, f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -380,10 +442,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 180: sub %o4, 8, %g2 - FREG_LOAD_1(%g2, f0) --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -- EX_LD_FP(LOAD_BLK(%o4, %f16)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) -+ EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) - FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) - FREG_MOVE_1(f30) - subcc %g1, 64, %g1 - add %o4, 64, %o4 -@@ -393,10 +455,10 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - nop - - 190: --1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3)) -+1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) - subcc %g1, 64, %g1 -- EX_LD_FP(LOAD_BLK(%o4, %f0)) -- EX_ST_FP(STORE_BLK(%f0, %o4 + %g3)) -+ EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) -+ EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) - add %o4, 64, %o4 - bne,pt %xcc, 1b - LOAD(prefetch, %o4 + 64, #one_read) -@@ -423,28 +485,28 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andn %o2, 0xf, %o4 - and %o2, 0xf, %o2 - 1: subcc %o4, 0x10, %o4 -- EX_LD(LOAD(ldx, %o1, %o5)) -+ EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) - add %o1, 0x08, %o1 -- EX_LD(LOAD(ldx, %o1, %g1)) -+ EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) - sub %o1, 0x08, %o1 -- EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) -+ EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) - add %o1, 0x8, %o1 -- EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) -+ EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 - 73: andcc %o2, 0x8, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x8, %o2 -- EX_LD(LOAD(ldx, %o1, %o5)) -- EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) -+ EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) -+ EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) - add %o1, 0x8, %o1 - 1: andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 -- EX_LD(LOAD(lduw, %o1, %o5)) -- EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) -+ EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) -+ EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) - add %o1, 0x4, %o1 - 1: cmp %o2, 0 - be,pt %XCC, 85f -@@ -460,8 +522,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %o2, %g1, %o2 - - 1: subcc %g1, 1, %g1 -- EX_LD(LOAD(ldub, %o1, %o5)) -- EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) -+ EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) -+ EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) - bgu,pt %icc, 1b - add %o1, 1, %o1 - -@@ -477,16 +539,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 8: mov 64, GLOBAL_SPARE - andn %o1, 0x7, %o1 -- EX_LD(LOAD(ldx, %o1, %g2)) -+ EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) - sub GLOBAL_SPARE, %g1, GLOBAL_SPARE - andn %o2, 0x7, %o4 - sllx %g2, %g1, %g2 - 1: add %o1, 0x8, %o1 -- EX_LD(LOAD(ldx, %o1, %g3)) -+ EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) - subcc %o4, 0x8, %o4 - srlx %g3, GLOBAL_SPARE, %o5 - or %o5, %g2, %o5 -- EX_ST(STORE(stx, %o5, %o0)) -+ EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 -@@ -506,8 +568,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 1: - subcc %o2, 4, %o2 -- EX_LD(LOAD(lduw, %o1, %g1)) -- EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) -+ EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) -+ EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -@@ -517,8 +579,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - .align 32 - 90: - subcc %o2, 1, %o2 -- EX_LD(LOAD(ldub, %o1, %g1)) -- EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) -+ EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) -+ EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl -diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S -index 2e8ee7a..16a286c 100644 ---- a/arch/sparc/lib/NG4copy_from_user.S -+++ b/arch/sparc/lib/NG4copy_from_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 2012 David S. Miller (davem@davemloft.net) - */ - --#define EX_LD(x) \ -+#define EX_LD(x, y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_LD_FP(x) \ -+#define EX_LD_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S -index be0bf45..6b0276f 100644 ---- a/arch/sparc/lib/NG4copy_to_user.S -+++ b/arch/sparc/lib/NG4copy_to_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 2012 David S. Miller (davem@davemloft.net) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_ST_FP(x) \ -+#define EX_ST_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_asi_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S -index 8e13ee1..75bb93b 100644 ---- a/arch/sparc/lib/NG4memcpy.S -+++ b/arch/sparc/lib/NG4memcpy.S -@@ -4,6 +4,7 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #include <asm/visasm.h> - #include <asm/asi.h> - #define GLOBAL_SPARE %g7 -@@ -46,22 +47,19 @@ - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - #ifndef EX_LD_FP --#define EX_LD_FP(x) x -+#define EX_LD_FP(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x -+#define EX_ST(x,y) x - #endif - #ifndef EX_ST_FP --#define EX_ST_FP(x) x -+#define EX_ST_FP(x,y) x - #endif - --#ifndef EX_RETVAL --#define EX_RETVAL(x) x --#endif - - #ifndef LOAD - #define LOAD(type,addr,dest) type [addr], dest -@@ -94,6 +92,158 @@ - .register %g3,#scratch - - .text -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+__restore_asi_fp: -+ VISExitHalf -+__restore_asi: -+ retl -+ wr %g0, ASI_AIUS, %asi -+ -+ENTRY(NG4_retl_o2) -+ ba,pt %xcc, __restore_asi -+ mov %o2, %o0 -+ENDPROC(NG4_retl_o2) -+ENTRY(NG4_retl_o2_plus_1) -+ ba,pt %xcc, __restore_asi -+ add %o2, 1, %o0 -+ENDPROC(NG4_retl_o2_plus_1) -+ENTRY(NG4_retl_o2_plus_4) -+ ba,pt %xcc, __restore_asi -+ add %o2, 4, %o0 -+ENDPROC(NG4_retl_o2_plus_4) -+ENTRY(NG4_retl_o2_plus_o5) -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5) -+ENTRY(NG4_retl_o2_plus_o5_plus_4) -+ add %o5, 4, %o5 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5_plus_4) -+ENTRY(NG4_retl_o2_plus_o5_plus_8) -+ add %o5, 8, %o5 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5_plus_8) -+ENTRY(NG4_retl_o2_plus_o5_plus_16) -+ add %o5, 16, %o5 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5_plus_16) -+ENTRY(NG4_retl_o2_plus_o5_plus_24) -+ add %o5, 24, %o5 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5_plus_24) -+ENTRY(NG4_retl_o2_plus_o5_plus_32) -+ add %o5, 32, %o5 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o5, %o0 -+ENDPROC(NG4_retl_o2_plus_o5_plus_32) -+ENTRY(NG4_retl_o2_plus_g1) -+ ba,pt %xcc, __restore_asi -+ add %o2, %g1, %o0 -+ENDPROC(NG4_retl_o2_plus_g1) -+ENTRY(NG4_retl_o2_plus_g1_plus_1) -+ add %g1, 1, %g1 -+ ba,pt %xcc, __restore_asi -+ add %o2, %g1, %o0 -+ENDPROC(NG4_retl_o2_plus_g1_plus_1) -+ENTRY(NG4_retl_o2_plus_g1_plus_8) -+ add %g1, 8, %g1 -+ ba,pt %xcc, __restore_asi -+ add %o2, %g1, %o0 -+ENDPROC(NG4_retl_o2_plus_g1_plus_8) -+ENTRY(NG4_retl_o2_plus_o4) -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4) -+ENTRY(NG4_retl_o2_plus_o4_plus_8) -+ add %o4, 8, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_8) -+ENTRY(NG4_retl_o2_plus_o4_plus_16) -+ add %o4, 16, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_16) -+ENTRY(NG4_retl_o2_plus_o4_plus_24) -+ add %o4, 24, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_24) -+ENTRY(NG4_retl_o2_plus_o4_plus_32) -+ add %o4, 32, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_32) -+ENTRY(NG4_retl_o2_plus_o4_plus_40) -+ add %o4, 40, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_40) -+ENTRY(NG4_retl_o2_plus_o4_plus_48) -+ add %o4, 48, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_48) -+ENTRY(NG4_retl_o2_plus_o4_plus_56) -+ add %o4, 56, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_56) -+ENTRY(NG4_retl_o2_plus_o4_plus_64) -+ add %o4, 64, %o4 -+ ba,pt %xcc, __restore_asi -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_64) -+ENTRY(NG4_retl_o2_plus_o4_fp) -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_8_fp) -+ add %o4, 8, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_8_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_16_fp) -+ add %o4, 16, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_16_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_24_fp) -+ add %o4, 24, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_24_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_32_fp) -+ add %o4, 32, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_32_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_40_fp) -+ add %o4, 40, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_40_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_48_fp) -+ add %o4, 48, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_48_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_56_fp) -+ add %o4, 56, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_56_fp) -+ENTRY(NG4_retl_o2_plus_o4_plus_64_fp) -+ add %o4, 64, %o4 -+ ba,pt %xcc, __restore_asi_fp -+ add %o2, %o4, %o0 -+ENDPROC(NG4_retl_o2_plus_o4_plus_64_fp) -+#endif - .align 64 - - .globl FUNC_NAME -@@ -124,12 +274,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - brz,pt %g1, 51f - sub %o2, %g1, %o2 - --1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) -+ -+1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) - add %o1, 1, %o1 - subcc %g1, 1, %g1 - add %o0, 1, %o0 - bne,pt %icc, 1b -- EX_ST(STORE(stb, %g2, %o0 - 0x01)) -+ EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) - - 51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) - LOAD(prefetch, %o1 + 0x080, #n_reads_strong) -@@ -154,43 +305,43 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - brz,pt %g1, .Llarge_aligned - sub %o2, %g1, %o2 - --1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) -+1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) - add %o1, 8, %o1 - subcc %g1, 8, %g1 - add %o0, 8, %o0 - bne,pt %icc, 1b -- EX_ST(STORE(stx, %g2, %o0 - 0x08)) -+ EX_ST(STORE(stx, %g2, %o0 - 0x08), NG4_retl_o2_plus_g1_plus_8) - - .Llarge_aligned: - /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ - andn %o2, 0x3f, %o4 - sub %o2, %o4, %o2 - --1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) -+1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o4) - add %o1, 0x40, %o1 -- EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) -+ EX_LD(LOAD(ldx, %o1 - 0x38, %g2), NG4_retl_o2_plus_o4) - subcc %o4, 0x40, %o4 -- EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) -- EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) -- EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) -- EX_ST(STORE_INIT(%g1, %o0)) -+ EX_LD(LOAD(ldx, %o1 - 0x30, %g3), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD(LOAD(ldx, %o1 - 0x20, %o5), NG4_retl_o2_plus_o4_plus_64) -+ EX_ST(STORE_INIT(%g1, %o0), NG4_retl_o2_plus_o4_plus_64) - add %o0, 0x08, %o0 -- EX_ST(STORE_INIT(%g2, %o0)) -+ EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_56) - add %o0, 0x08, %o0 -- EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) -- EX_ST(STORE_INIT(%g3, %o0)) -+ EX_LD(LOAD(ldx, %o1 - 0x18, %g2), NG4_retl_o2_plus_o4_plus_48) -+ EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_48) - add %o0, 0x08, %o0 -- EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) -- EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) -+ EX_LD(LOAD(ldx, %o1 - 0x10, %g3), NG4_retl_o2_plus_o4_plus_40) -+ EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_40) - add %o0, 0x08, %o0 -- EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) -- EX_ST(STORE_INIT(%o5, %o0)) -+ EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), NG4_retl_o2_plus_o4_plus_32) -+ EX_ST(STORE_INIT(%o5, %o0), NG4_retl_o2_plus_o4_plus_32) - add %o0, 0x08, %o0 -- EX_ST(STORE_INIT(%g2, %o0)) -+ EX_ST(STORE_INIT(%g2, %o0), NG4_retl_o2_plus_o4_plus_24) - add %o0, 0x08, %o0 -- EX_ST(STORE_INIT(%g3, %o0)) -+ EX_ST(STORE_INIT(%g3, %o0), NG4_retl_o2_plus_o4_plus_16) - add %o0, 0x08, %o0 -- EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) -+ EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), NG4_retl_o2_plus_o4_plus_8) - add %o0, 0x08, %o0 - bne,pt %icc, 1b - LOAD(prefetch, %o1 + 0x200, #n_reads_strong) -@@ -216,17 +367,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %o2, %o4, %o2 - alignaddr %o1, %g0, %g1 - add %o1, %o4, %o1 -- EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0)) --1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2)) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), NG4_retl_o2_plus_o4) -+1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), NG4_retl_o2_plus_o4) - subcc %o4, 0x40, %o4 -- EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4)) -- EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6)) -- EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8)) -- EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10)) -- EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12)) -- EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14)) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), NG4_retl_o2_plus_o4_plus_64) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), NG4_retl_o2_plus_o4_plus_64) - faligndata %f0, %f2, %f16 -- EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0)) -+ EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), NG4_retl_o2_plus_o4_plus_64) - faligndata %f2, %f4, %f18 - add %g1, 0x40, %g1 - faligndata %f4, %f6, %f20 -@@ -235,14 +386,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - faligndata %f10, %f12, %f26 - faligndata %f12, %f14, %f28 - faligndata %f14, %f0, %f30 -- EX_ST_FP(STORE(std, %f16, %o0 + 0x00)) -- EX_ST_FP(STORE(std, %f18, %o0 + 0x08)) -- EX_ST_FP(STORE(std, %f20, %o0 + 0x10)) -- EX_ST_FP(STORE(std, %f22, %o0 + 0x18)) -- EX_ST_FP(STORE(std, %f24, %o0 + 0x20)) -- EX_ST_FP(STORE(std, %f26, %o0 + 0x28)) -- EX_ST_FP(STORE(std, %f28, %o0 + 0x30)) -- EX_ST_FP(STORE(std, %f30, %o0 + 0x38)) -+ EX_ST_FP(STORE(std, %f16, %o0 + 0x00), NG4_retl_o2_plus_o4_plus_64) -+ EX_ST_FP(STORE(std, %f18, %o0 + 0x08), NG4_retl_o2_plus_o4_plus_56) -+ EX_ST_FP(STORE(std, %f20, %o0 + 0x10), NG4_retl_o2_plus_o4_plus_48) -+ EX_ST_FP(STORE(std, %f22, %o0 + 0x18), NG4_retl_o2_plus_o4_plus_40) -+ EX_ST_FP(STORE(std, %f24, %o0 + 0x20), NG4_retl_o2_plus_o4_plus_32) -+ EX_ST_FP(STORE(std, %f26, %o0 + 0x28), NG4_retl_o2_plus_o4_plus_24) -+ EX_ST_FP(STORE(std, %f28, %o0 + 0x30), NG4_retl_o2_plus_o4_plus_16) -+ EX_ST_FP(STORE(std, %f30, %o0 + 0x38), NG4_retl_o2_plus_o4_plus_8) - add %o0, 0x40, %o0 - bne,pt %icc, 1b - LOAD(prefetch, %g1 + 0x200, #n_reads_strong) -@@ -270,37 +421,38 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andncc %o2, 0x20 - 1, %o5 - be,pn %icc, 2f - sub %o2, %o5, %o2 --1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) -- EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) -- EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) -- EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) -+1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) -+ EX_LD(LOAD(ldx, %o1 + 0x08, %g2), NG4_retl_o2_plus_o5) -+ EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), NG4_retl_o2_plus_o5) -+ EX_LD(LOAD(ldx, %o1 + 0x18, %o4), NG4_retl_o2_plus_o5) - add %o1, 0x20, %o1 - subcc %o5, 0x20, %o5 -- EX_ST(STORE(stx, %g1, %o0 + 0x00)) -- EX_ST(STORE(stx, %g2, %o0 + 0x08)) -- EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) -- EX_ST(STORE(stx, %o4, %o0 + 0x18)) -+ EX_ST(STORE(stx, %g1, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_32) -+ EX_ST(STORE(stx, %g2, %o0 + 0x08), NG4_retl_o2_plus_o5_plus_24) -+ EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), NG4_retl_o2_plus_o5_plus_24) -+ EX_ST(STORE(stx, %o4, %o0 + 0x18), NG4_retl_o2_plus_o5_plus_8) - bne,pt %icc, 1b - add %o0, 0x20, %o0 - 2: andcc %o2, 0x18, %o5 - be,pt %icc, 3f - sub %o2, %o5, %o2 --1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) -+ -+1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) - add %o1, 0x08, %o1 - add %o0, 0x08, %o0 - subcc %o5, 0x08, %o5 - bne,pt %icc, 1b -- EX_ST(STORE(stx, %g1, %o0 - 0x08)) -+ EX_ST(STORE(stx, %g1, %o0 - 0x08), NG4_retl_o2_plus_o5_plus_8) - 3: brz,pt %o2, .Lexit - cmp %o2, 0x04 - bl,pn %icc, .Ltiny - nop -- EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) -+ EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2) - add %o1, 0x04, %o1 - add %o0, 0x04, %o0 - subcc %o2, 0x04, %o2 - bne,pn %icc, .Ltiny -- EX_ST(STORE(stw, %g1, %o0 - 0x04)) -+ EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_4) - ba,a,pt %icc, .Lexit - .Lmedium_unaligned: - /* First get dest 8 byte aligned. */ -@@ -309,12 +461,12 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - brz,pt %g1, 2f - sub %o2, %g1, %o2 - --1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) -+1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), NG4_retl_o2_plus_g1) - add %o1, 1, %o1 - subcc %g1, 1, %g1 - add %o0, 1, %o0 - bne,pt %icc, 1b -- EX_ST(STORE(stb, %g2, %o0 - 0x01)) -+ EX_ST(STORE(stb, %g2, %o0 - 0x01), NG4_retl_o2_plus_g1_plus_1) - 2: - and %o1, 0x7, %g1 - brz,pn %g1, .Lmedium_noprefetch -@@ -322,16 +474,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - mov 64, %g2 - sub %g2, %g1, %g2 - andn %o1, 0x7, %o1 -- EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) -+ EX_LD(LOAD(ldx, %o1 + 0x00, %o4), NG4_retl_o2) - sllx %o4, %g1, %o4 - andn %o2, 0x08 - 1, %o5 - sub %o2, %o5, %o2 --1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) -+1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), NG4_retl_o2_plus_o5) - add %o1, 0x08, %o1 - subcc %o5, 0x08, %o5 - srlx %g3, %g2, GLOBAL_SPARE - or GLOBAL_SPARE, %o4, GLOBAL_SPARE -- EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) -+ EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), NG4_retl_o2_plus_o5_plus_8) - add %o0, 0x08, %o0 - bne,pt %icc, 1b - sllx %g3, %g1, %o4 -@@ -342,17 +494,17 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - ba,pt %icc, .Lsmall_unaligned - - .Ltiny: -- EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) -+ EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) - subcc %o2, 1, %o2 - be,pn %icc, .Lexit -- EX_ST(STORE(stb, %g1, %o0 + 0x00)) -- EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) -+ EX_ST(STORE(stb, %g1, %o0 + 0x00), NG4_retl_o2_plus_1) -+ EX_LD(LOAD(ldub, %o1 + 0x01, %g1), NG4_retl_o2) - subcc %o2, 1, %o2 - be,pn %icc, .Lexit -- EX_ST(STORE(stb, %g1, %o0 + 0x01)) -- EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) -+ EX_ST(STORE(stb, %g1, %o0 + 0x01), NG4_retl_o2_plus_1) -+ EX_LD(LOAD(ldub, %o1 + 0x02, %g1), NG4_retl_o2) - ba,pt %icc, .Lexit -- EX_ST(STORE(stb, %g1, %o0 + 0x02)) -+ EX_ST(STORE(stb, %g1, %o0 + 0x02), NG4_retl_o2) - - .Lsmall: - andcc %g2, 0x3, %g0 -@@ -360,22 +512,22 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andn %o2, 0x4 - 1, %o5 - sub %o2, %o5, %o2 - 1: -- EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) -+ EX_LD(LOAD(lduw, %o1 + 0x00, %g1), NG4_retl_o2_plus_o5) - add %o1, 0x04, %o1 - subcc %o5, 0x04, %o5 - add %o0, 0x04, %o0 - bne,pt %icc, 1b -- EX_ST(STORE(stw, %g1, %o0 - 0x04)) -+ EX_ST(STORE(stw, %g1, %o0 - 0x04), NG4_retl_o2_plus_o5_plus_4) - brz,pt %o2, .Lexit - nop - ba,a,pt %icc, .Ltiny - - .Lsmall_unaligned: --1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) -+1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), NG4_retl_o2) - add %o1, 1, %o1 - add %o0, 1, %o0 - subcc %o2, 1, %o2 - bne,pt %icc, 1b -- EX_ST(STORE(stb, %g1, %o0 - 0x01)) -+ EX_ST(STORE(stb, %g1, %o0 - 0x01), NG4_retl_o2_plus_1) - ba,a,pt %icc, .Lexit - .size FUNC_NAME, .-FUNC_NAME -diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S -index 5d1e4d1..9cd42fc 100644 ---- a/arch/sparc/lib/NGcopy_from_user.S -+++ b/arch/sparc/lib/NGcopy_from_user.S -@@ -3,11 +3,11 @@ - * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_LD(x) \ -+#define EX_LD(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __ret_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S -index ff630dc..5c358af 100644 ---- a/arch/sparc/lib/NGcopy_to_user.S -+++ b/arch/sparc/lib/NGcopy_to_user.S -@@ -3,11 +3,11 @@ - * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __ret_one_asi;\ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S -index 96a14ca..d88c4ed 100644 ---- a/arch/sparc/lib/NGmemcpy.S -+++ b/arch/sparc/lib/NGmemcpy.S -@@ -4,6 +4,7 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #include <asm/asi.h> - #include <asm/thread_info.h> - #define GLOBAL_SPARE %g7 -@@ -27,15 +28,11 @@ - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x --#endif -- --#ifndef EX_RETVAL --#define EX_RETVAL(x) x -+#define EX_ST(x,y) x - #endif - - #ifndef LOAD -@@ -79,6 +76,92 @@ - .register %g3,#scratch - - .text -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+__restore_asi: -+ ret -+ wr %g0, ASI_AIUS, %asi -+ restore -+ENTRY(NG_ret_i2_plus_i4_plus_1) -+ ba,pt %xcc, __restore_asi -+ add %i2, %i5, %i0 -+ENDPROC(NG_ret_i2_plus_i4_plus_1) -+ENTRY(NG_ret_i2_plus_g1) -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1) -+ENTRY(NG_ret_i2_plus_g1_minus_8) -+ sub %g1, 8, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_8) -+ENTRY(NG_ret_i2_plus_g1_minus_16) -+ sub %g1, 16, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_16) -+ENTRY(NG_ret_i2_plus_g1_minus_24) -+ sub %g1, 24, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_24) -+ENTRY(NG_ret_i2_plus_g1_minus_32) -+ sub %g1, 32, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_32) -+ENTRY(NG_ret_i2_plus_g1_minus_40) -+ sub %g1, 40, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_40) -+ENTRY(NG_ret_i2_plus_g1_minus_48) -+ sub %g1, 48, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_48) -+ENTRY(NG_ret_i2_plus_g1_minus_56) -+ sub %g1, 56, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_minus_56) -+ENTRY(NG_ret_i2_plus_i4) -+ ba,pt %xcc, __restore_asi -+ add %i2, %i4, %i0 -+ENDPROC(NG_ret_i2_plus_i4) -+ENTRY(NG_ret_i2_plus_i4_minus_8) -+ sub %i4, 8, %i4 -+ ba,pt %xcc, __restore_asi -+ add %i2, %i4, %i0 -+ENDPROC(NG_ret_i2_plus_i4_minus_8) -+ENTRY(NG_ret_i2_plus_8) -+ ba,pt %xcc, __restore_asi -+ add %i2, 8, %i0 -+ENDPROC(NG_ret_i2_plus_8) -+ENTRY(NG_ret_i2_plus_4) -+ ba,pt %xcc, __restore_asi -+ add %i2, 4, %i0 -+ENDPROC(NG_ret_i2_plus_4) -+ENTRY(NG_ret_i2_plus_1) -+ ba,pt %xcc, __restore_asi -+ add %i2, 1, %i0 -+ENDPROC(NG_ret_i2_plus_1) -+ENTRY(NG_ret_i2_plus_g1_plus_1) -+ add %g1, 1, %g1 -+ ba,pt %xcc, __restore_asi -+ add %i2, %g1, %i0 -+ENDPROC(NG_ret_i2_plus_g1_plus_1) -+ENTRY(NG_ret_i2) -+ ba,pt %xcc, __restore_asi -+ mov %i2, %i0 -+ENDPROC(NG_ret_i2) -+ENTRY(NG_ret_i2_and_7_plus_i4) -+ and %i2, 7, %i2 -+ ba,pt %xcc, __restore_asi -+ add %i2, %i4, %i0 -+ENDPROC(NG_ret_i2_and_7_plus_i4) -+#endif -+ - .align 64 - - .globl FUNC_NAME -@@ -126,8 +209,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - sub %g0, %i4, %i4 ! bytes to align dst - sub %i2, %i4, %i2 - 1: subcc %i4, 1, %i4 -- EX_LD(LOAD(ldub, %i1, %g1)) -- EX_ST(STORE(stb, %g1, %o0)) -+ EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) -+ EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) - add %i1, 1, %i1 - bne,pt %XCC, 1b - add %o0, 1, %o0 -@@ -160,7 +243,7 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - and %i4, 0x7, GLOBAL_SPARE - sll GLOBAL_SPARE, 3, GLOBAL_SPARE - mov 64, %i5 -- EX_LD(LOAD_TWIN(%i1, %g2, %g3)) -+ EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) - sub %i5, GLOBAL_SPARE, %i5 - mov 16, %o4 - mov 32, %o5 -@@ -178,31 +261,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - srlx WORD3, PRE_SHIFT, TMP; \ - or WORD2, TMP, WORD2; - --8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) -+8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) - MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) - LOAD(prefetch, %i1 + %i3, #one_read) - -- EX_ST(STORE_INIT(%g2, %o0 + 0x00)) -- EX_ST(STORE_INIT(%g3, %o0 + 0x08)) -+ EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) -+ EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - -- EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) -+ EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) - MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%o2, %o0 + 0x10)) -- EX_ST(STORE_INIT(%o3, %o0 + 0x18)) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - -- EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) -+ EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) - MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%g2, %o0 + 0x20)) -- EX_ST(STORE_INIT(%g3, %o0 + 0x28)) -+ EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) -+ EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - -- EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) -+ EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) - add %i1, 64, %i1 - MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%o2, %o0 + 0x30)) -- EX_ST(STORE_INIT(%o3, %o0 + 0x38)) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) - - subcc %g1, 64, %g1 - bne,pt %XCC, 8b -@@ -211,31 +294,31 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - ba,pt %XCC, 60f - add %i1, %i4, %i1 - --9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3)) -+9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) - MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) - LOAD(prefetch, %i1 + %i3, #one_read) - -- EX_ST(STORE_INIT(%g3, %o0 + 0x00)) -- EX_ST(STORE_INIT(%o2, %o0 + 0x08)) -+ EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) - -- EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3)) -+ EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) - MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%o3, %o0 + 0x10)) -- EX_ST(STORE_INIT(%g2, %o0 + 0x18)) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) - -- EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) -+ EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) - MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%g3, %o0 + 0x20)) -- EX_ST(STORE_INIT(%o2, %o0 + 0x28)) -+ EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) - -- EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3)) -+ EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) - add %i1, 64, %i1 - MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) - -- EX_ST(STORE_INIT(%o3, %o0 + 0x30)) -- EX_ST(STORE_INIT(%g2, %o0 + 0x38)) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) -+ EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) - - subcc %g1, 64, %g1 - bne,pt %XCC, 9b -@@ -249,25 +332,25 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - * one twin load ahead, then add 8 back into source when - * we finish the loop. - */ -- EX_LD(LOAD_TWIN(%i1, %o4, %o5)) -+ EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) - mov 16, %o7 - mov 32, %g2 - mov 48, %g3 - mov 64, %o1 --1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) -+1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) - LOAD(prefetch, %i1 + %o1, #one_read) -- EX_ST(STORE_INIT(%o5, %o0 + 0x00)) ! initializes cache line -- EX_ST(STORE_INIT(%o2, %o0 + 0x08)) -- EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) -- EX_ST(STORE_INIT(%o3, %o0 + 0x10)) -- EX_ST(STORE_INIT(%o4, %o0 + 0x18)) -- EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) -- EX_ST(STORE_INIT(%o5, %o0 + 0x20)) -- EX_ST(STORE_INIT(%o2, %o0 + 0x28)) -- EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5)) -+ EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line -+ EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) -+ EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) -+ EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) -+ EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) -+ EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) - add %i1, 64, %i1 -- EX_ST(STORE_INIT(%o3, %o0 + 0x30)) -- EX_ST(STORE_INIT(%o4, %o0 + 0x38)) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) -+ EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) - subcc %g1, 64, %g1 - bne,pt %XCC, 1b - add %o0, 64, %o0 -@@ -282,20 +365,20 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - mov 32, %g2 - mov 48, %g3 - mov 64, %o1 --1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5)) -- EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3)) -+1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) -+ EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) - LOAD(prefetch, %i1 + %o1, #one_read) -- EX_ST(STORE_INIT(%o4, %o0 + 0x00)) ! initializes cache line -- EX_ST(STORE_INIT(%o5, %o0 + 0x08)) -- EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5)) -- EX_ST(STORE_INIT(%o2, %o0 + 0x10)) -- EX_ST(STORE_INIT(%o3, %o0 + 0x18)) -- EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3)) -+ EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line -+ EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) -+ EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) -+ EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) - add %i1, 64, %i1 -- EX_ST(STORE_INIT(%o4, %o0 + 0x20)) -- EX_ST(STORE_INIT(%o5, %o0 + 0x28)) -- EX_ST(STORE_INIT(%o2, %o0 + 0x30)) -- EX_ST(STORE_INIT(%o3, %o0 + 0x38)) -+ EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) -+ EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) -+ EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) -+ EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) - subcc %g1, 64, %g1 - bne,pt %XCC, 1b - add %o0, 64, %o0 -@@ -321,28 +404,28 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - andn %i2, 0xf, %i4 - and %i2, 0xf, %i2 - 1: subcc %i4, 0x10, %i4 -- EX_LD(LOAD(ldx, %i1, %o4)) -+ EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4) - add %i1, 0x08, %i1 -- EX_LD(LOAD(ldx, %i1, %g1)) -+ EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4) - sub %i1, 0x08, %i1 -- EX_ST(STORE(stx, %o4, %i1 + %i3)) -+ EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4) - add %i1, 0x8, %i1 -- EX_ST(STORE(stx, %g1, %i1 + %i3)) -+ EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_minus_8) - bgu,pt %XCC, 1b - add %i1, 0x8, %i1 - 73: andcc %i2, 0x8, %g0 - be,pt %XCC, 1f - nop - sub %i2, 0x8, %i2 -- EX_LD(LOAD(ldx, %i1, %o4)) -- EX_ST(STORE(stx, %o4, %i1 + %i3)) -+ EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) -+ EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) - add %i1, 0x8, %i1 - 1: andcc %i2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %i2, 0x4, %i2 -- EX_LD(LOAD(lduw, %i1, %i5)) -- EX_ST(STORE(stw, %i5, %i1 + %i3)) -+ EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) -+ EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) - add %i1, 0x4, %i1 - 1: cmp %i2, 0 - be,pt %XCC, 85f -@@ -358,8 +441,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - sub %i2, %g1, %i2 - - 1: subcc %g1, 1, %g1 -- EX_LD(LOAD(ldub, %i1, %i5)) -- EX_ST(STORE(stb, %i5, %i1 + %i3)) -+ EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) -+ EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) - bgu,pt %icc, 1b - add %i1, 1, %i1 - -@@ -375,16 +458,16 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - - 8: mov 64, %i3 - andn %i1, 0x7, %i1 -- EX_LD(LOAD(ldx, %i1, %g2)) -+ EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) - sub %i3, %g1, %i3 - andn %i2, 0x7, %i4 - sllx %g2, %g1, %g2 - 1: add %i1, 0x8, %i1 -- EX_LD(LOAD(ldx, %i1, %g3)) -+ EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) - subcc %i4, 0x8, %i4 - srlx %g3, %i3, %i5 - or %i5, %g2, %i5 -- EX_ST(STORE(stx, %i5, %o0)) -+ EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 -@@ -404,8 +487,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - - 1: - subcc %i2, 4, %i2 -- EX_LD(LOAD(lduw, %i1, %g1)) -- EX_ST(STORE(stw, %g1, %i1 + %i3)) -+ EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) -+ EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) - bgu,pt %XCC, 1b - add %i1, 4, %i1 - -@@ -415,8 +498,8 @@ FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ - .align 32 - 90: - subcc %i2, 1, %i2 -- EX_LD(LOAD(ldub, %i1, %g1)) -- EX_ST(STORE(stb, %g1, %i1 + %i3)) -+ EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) -+ EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) - bgu,pt %XCC, 90b - add %i1, 1, %i1 - ret -diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S -index ecc5692..bb6ff73 100644 ---- a/arch/sparc/lib/U1copy_from_user.S -+++ b/arch/sparc/lib/U1copy_from_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) - */ - --#define EX_LD(x) \ -+#define EX_LD(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_LD_FP(x) \ -+#define EX_LD_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_fp;\ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S -index 9eea392..ed92ce73 100644 ---- a/arch/sparc/lib/U1copy_to_user.S -+++ b/arch/sparc/lib/U1copy_to_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_ST_FP(x) \ -+#define EX_ST_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_fp;\ -+ .word 98b, y; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S -index 3e6209e..f30d2ab 100644 ---- a/arch/sparc/lib/U1memcpy.S -+++ b/arch/sparc/lib/U1memcpy.S -@@ -5,6 +5,7 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #include <asm/visasm.h> - #include <asm/asi.h> - #define GLOBAL_SPARE g7 -@@ -23,21 +24,17 @@ - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - #ifndef EX_LD_FP --#define EX_LD_FP(x) x -+#define EX_LD_FP(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x -+#define EX_ST(x,y) x - #endif - #ifndef EX_ST_FP --#define EX_ST_FP(x) x --#endif -- --#ifndef EX_RETVAL --#define EX_RETVAL(x) x -+#define EX_ST_FP(x,y) x - #endif - - #ifndef LOAD -@@ -78,53 +75,169 @@ - faligndata %f7, %f8, %f60; \ - faligndata %f8, %f9, %f62; - --#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ -- EX_LD_FP(LOAD_BLK(%src, %fdest)); \ -- EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ -- add %src, 0x40, %src; \ -- subcc %len, 0x40, %len; \ -- be,pn %xcc, jmptgt; \ -- add %dest, 0x40, %dest; \ -- --#define LOOP_CHUNK1(src, dest, len, branch_dest) \ -- MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) --#define LOOP_CHUNK2(src, dest, len, branch_dest) \ -- MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) --#define LOOP_CHUNK3(src, dest, len, branch_dest) \ -- MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) -+#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ -+ EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ -+ EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ -+ add %src, 0x40, %src; \ -+ subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ -+ be,pn %xcc, jmptgt; \ -+ add %dest, 0x40, %dest; \ -+ -+#define LOOP_CHUNK1(src, dest, branch_dest) \ -+ MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) -+#define LOOP_CHUNK2(src, dest, branch_dest) \ -+ MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) -+#define LOOP_CHUNK3(src, dest, branch_dest) \ -+ MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) - - #define DO_SYNC membar #Sync; - #define STORE_SYNC(dest, fsrc) \ -- EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ -+ EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ - add %dest, 0x40, %dest; \ - DO_SYNC - - #define STORE_JUMP(dest, fsrc, target) \ -- EX_ST_FP(STORE_BLK(%fsrc, %dest)); \ -+ EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ - add %dest, 0x40, %dest; \ - ba,pt %xcc, target; \ - nop; - --#define FINISH_VISCHUNK(dest, f0, f1, left) \ -- subcc %left, 8, %left;\ -- bl,pn %xcc, 95f; \ -- faligndata %f0, %f1, %f48; \ -- EX_ST_FP(STORE(std, %f48, %dest)); \ -+#define FINISH_VISCHUNK(dest, f0, f1) \ -+ subcc %g3, 8, %g3; \ -+ bl,pn %xcc, 95f; \ -+ faligndata %f0, %f1, %f48; \ -+ EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ - add %dest, 8, %dest; - --#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ -- subcc %left, 8, %left; \ -- bl,pn %xcc, 95f; \ -+#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ -+ subcc %g3, 8, %g3; \ -+ bl,pn %xcc, 95f; \ - fsrc2 %f0, %f1; - --#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ -- UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ -+#define UNEVEN_VISCHUNK(dest, f0, f1) \ -+ UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ - ba,a,pt %xcc, 93f; - - .register %g2,#scratch - .register %g3,#scratch - - .text -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+ENTRY(U1_g1_1_fp) -+ VISExitHalf -+ add %g1, 1, %g1 -+ add %g1, %g2, %g1 -+ retl -+ add %g1, %o2, %o0 -+ENDPROC(U1_g1_1_fp) -+ENTRY(U1_g2_0_fp) -+ VISExitHalf -+ retl -+ add %g2, %o2, %o0 -+ENDPROC(U1_g2_0_fp) -+ENTRY(U1_g2_8_fp) -+ VISExitHalf -+ add %g2, 8, %g2 -+ retl -+ add %g2, %o2, %o0 -+ENDPROC(U1_g2_8_fp) -+ENTRY(U1_gs_0_fp) -+ VISExitHalf -+ add %GLOBAL_SPARE, %g3, %o0 -+ retl -+ add %o0, %o2, %o0 -+ENDPROC(U1_gs_0_fp) -+ENTRY(U1_gs_80_fp) -+ VISExitHalf -+ add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE -+ add %GLOBAL_SPARE, %g3, %o0 -+ retl -+ add %o0, %o2, %o0 -+ENDPROC(U1_gs_80_fp) -+ENTRY(U1_gs_40_fp) -+ VISExitHalf -+ add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE -+ add %GLOBAL_SPARE, %g3, %o0 -+ retl -+ add %o0, %o2, %o0 -+ENDPROC(U1_gs_40_fp) -+ENTRY(U1_g3_0_fp) -+ VISExitHalf -+ retl -+ add %g3, %o2, %o0 -+ENDPROC(U1_g3_0_fp) -+ENTRY(U1_g3_8_fp) -+ VISExitHalf -+ add %g3, 8, %g3 -+ retl -+ add %g3, %o2, %o0 -+ENDPROC(U1_g3_8_fp) -+ENTRY(U1_o2_0_fp) -+ VISExitHalf -+ retl -+ mov %o2, %o0 -+ENDPROC(U1_o2_0_fp) -+ENTRY(U1_o2_1_fp) -+ VISExitHalf -+ retl -+ add %o2, 1, %o0 -+ENDPROC(U1_o2_1_fp) -+ENTRY(U1_gs_0) -+ VISExitHalf -+ retl -+ add %GLOBAL_SPARE, %o2, %o0 -+ENDPROC(U1_gs_0) -+ENTRY(U1_gs_8) -+ VISExitHalf -+ add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE -+ retl -+ add %GLOBAL_SPARE, 0x8, %o0 -+ENDPROC(U1_gs_8) -+ENTRY(U1_gs_10) -+ VISExitHalf -+ add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE -+ retl -+ add %GLOBAL_SPARE, 0x10, %o0 -+ENDPROC(U1_gs_10) -+ENTRY(U1_o2_0) -+ retl -+ mov %o2, %o0 -+ENDPROC(U1_o2_0) -+ENTRY(U1_o2_8) -+ retl -+ add %o2, 8, %o0 -+ENDPROC(U1_o2_8) -+ENTRY(U1_o2_4) -+ retl -+ add %o2, 4, %o0 -+ENDPROC(U1_o2_4) -+ENTRY(U1_o2_1) -+ retl -+ add %o2, 1, %o0 -+ENDPROC(U1_o2_1) -+ENTRY(U1_g1_0) -+ retl -+ add %g1, %o2, %o0 -+ENDPROC(U1_g1_0) -+ENTRY(U1_g1_1) -+ add %g1, 1, %g1 -+ retl -+ add %g1, %o2, %o0 -+ENDPROC(U1_g1_1) -+ENTRY(U1_gs_0_o2_adj) -+ and %o2, 7, %o2 -+ retl -+ add %GLOBAL_SPARE, %o2, %o0 -+ENDPROC(U1_gs_0_o2_adj) -+ENTRY(U1_gs_8_o2_adj) -+ and %o2, 7, %o2 -+ add %GLOBAL_SPARE, 8, %GLOBAL_SPARE -+ retl -+ add %GLOBAL_SPARE, %o2, %o0 -+ENDPROC(U1_gs_8_o2_adj) -+#endif -+ - .align 64 - - .globl FUNC_NAME -@@ -166,8 +279,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - and %g2, 0x38, %g2 - - 1: subcc %g1, 0x1, %g1 -- EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) -- EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) -+ EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) -+ EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) - bgu,pt %XCC, 1b - add %o1, 0x1, %o1 - -@@ -178,20 +291,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - be,pt %icc, 3f - alignaddr %o1, %g0, %o1 - -- EX_LD_FP(LOAD(ldd, %o1, %f4)) --1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) -+ EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) -+1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f4, %f6, %f0 -- EX_ST_FP(STORE(std, %f0, %o0)) -+ EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) - be,pn %icc, 3f - add %o0, 0x8, %o0 - -- EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f6, %f4, %f0 -- EX_ST_FP(STORE(std, %f0, %o0)) -+ EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) - bne,pt %icc, 1b - add %o0, 0x8, %o0 - -@@ -214,13 +327,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - add %g1, %GLOBAL_SPARE, %g1 - subcc %o2, %g3, %o2 - -- EX_LD_FP(LOAD_BLK(%o1, %f0)) -+ EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) - add %o1, 0x40, %o1 - add %g1, %g3, %g1 -- EX_LD_FP(LOAD_BLK(%o1, %f16)) -+ EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) - add %o1, 0x40, %o1 - sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE -- EX_LD_FP(LOAD_BLK(%o1, %f32)) -+ EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) - add %o1, 0x40, %o1 - - /* There are 8 instances of the unrolled loop, -@@ -240,11 +353,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - .align 64 - 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f0, %f2, %f48 - 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) -@@ -261,11 +374,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 56f) - - 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f2, %f4, %f48 - 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) -@@ -282,11 +395,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 57f) - - 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f4, %f6, %f48 - 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) -@@ -303,11 +416,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 58f) - - 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f6, %f8, %f48 - 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) -@@ -324,11 +437,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 59f) - - 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f8, %f10, %f48 - 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) -@@ -345,11 +458,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 60f) - - 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f10, %f12, %f48 - 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) -@@ -366,11 +479,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 61f) - - 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f12, %f14, %f48 - 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) -@@ -387,11 +500,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - STORE_JUMP(o0, f48, 62f) - - 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) -- LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) -+ LOOP_CHUNK1(o1, o0, 1f) - FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) -- LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) -+ LOOP_CHUNK2(o1, o0, 2f) - FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) -- LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) -+ LOOP_CHUNK3(o1, o0, 3f) - ba,pt %xcc, 1b+4 - faligndata %f14, %f16, %f48 - 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) -@@ -407,53 +520,53 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) - STORE_JUMP(o0, f48, 63f) - --40: FINISH_VISCHUNK(o0, f0, f2, g3) --41: FINISH_VISCHUNK(o0, f2, f4, g3) --42: FINISH_VISCHUNK(o0, f4, f6, g3) --43: FINISH_VISCHUNK(o0, f6, f8, g3) --44: FINISH_VISCHUNK(o0, f8, f10, g3) --45: FINISH_VISCHUNK(o0, f10, f12, g3) --46: FINISH_VISCHUNK(o0, f12, f14, g3) --47: UNEVEN_VISCHUNK(o0, f14, f0, g3) --48: FINISH_VISCHUNK(o0, f16, f18, g3) --49: FINISH_VISCHUNK(o0, f18, f20, g3) --50: FINISH_VISCHUNK(o0, f20, f22, g3) --51: FINISH_VISCHUNK(o0, f22, f24, g3) --52: FINISH_VISCHUNK(o0, f24, f26, g3) --53: FINISH_VISCHUNK(o0, f26, f28, g3) --54: FINISH_VISCHUNK(o0, f28, f30, g3) --55: UNEVEN_VISCHUNK(o0, f30, f0, g3) --56: FINISH_VISCHUNK(o0, f32, f34, g3) --57: FINISH_VISCHUNK(o0, f34, f36, g3) --58: FINISH_VISCHUNK(o0, f36, f38, g3) --59: FINISH_VISCHUNK(o0, f38, f40, g3) --60: FINISH_VISCHUNK(o0, f40, f42, g3) --61: FINISH_VISCHUNK(o0, f42, f44, g3) --62: FINISH_VISCHUNK(o0, f44, f46, g3) --63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) -- --93: EX_LD_FP(LOAD(ldd, %o1, %f2)) -+40: FINISH_VISCHUNK(o0, f0, f2) -+41: FINISH_VISCHUNK(o0, f2, f4) -+42: FINISH_VISCHUNK(o0, f4, f6) -+43: FINISH_VISCHUNK(o0, f6, f8) -+44: FINISH_VISCHUNK(o0, f8, f10) -+45: FINISH_VISCHUNK(o0, f10, f12) -+46: FINISH_VISCHUNK(o0, f12, f14) -+47: UNEVEN_VISCHUNK(o0, f14, f0) -+48: FINISH_VISCHUNK(o0, f16, f18) -+49: FINISH_VISCHUNK(o0, f18, f20) -+50: FINISH_VISCHUNK(o0, f20, f22) -+51: FINISH_VISCHUNK(o0, f22, f24) -+52: FINISH_VISCHUNK(o0, f24, f26) -+53: FINISH_VISCHUNK(o0, f26, f28) -+54: FINISH_VISCHUNK(o0, f28, f30) -+55: UNEVEN_VISCHUNK(o0, f30, f0) -+56: FINISH_VISCHUNK(o0, f32, f34) -+57: FINISH_VISCHUNK(o0, f34, f36) -+58: FINISH_VISCHUNK(o0, f36, f38) -+59: FINISH_VISCHUNK(o0, f38, f40) -+60: FINISH_VISCHUNK(o0, f40, f42) -+61: FINISH_VISCHUNK(o0, f42, f44) -+62: FINISH_VISCHUNK(o0, f44, f46) -+63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) -+ -+93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_0_fp) - add %o1, 8, %o1 - subcc %g3, 8, %g3 - faligndata %f0, %f2, %f8 -- EX_ST_FP(STORE(std, %f8, %o0)) -+ EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) - bl,pn %xcc, 95f - add %o0, 8, %o0 -- EX_LD_FP(LOAD(ldd, %o1, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_0_fp) - add %o1, 8, %o1 - subcc %g3, 8, %g3 - faligndata %f2, %f0, %f8 -- EX_ST_FP(STORE(std, %f8, %o0)) -+ EX_ST_FP(STORE(std, %f8, %o0), U1_g3_8_fp) - bge,pt %xcc, 93b - add %o0, 8, %o0 - - 95: brz,pt %o2, 2f - mov %g1, %o1 - --1: EX_LD_FP(LOAD(ldub, %o1, %o3)) -+1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) - add %o1, 1, %o1 - subcc %o2, 1, %o2 -- EX_ST_FP(STORE(stb, %o3, %o0)) -+ EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) - bne,pt %xcc, 1b - add %o0, 1, %o0 - -@@ -469,27 +582,27 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 72: andn %o2, 0xf, %GLOBAL_SPARE - and %o2, 0xf, %o2 --1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) -- EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) -+1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) -+ EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) - subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE -- EX_ST(STORE(stx, %o5, %o1 + %o3)) -+ EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) - add %o1, 0x8, %o1 -- EX_ST(STORE(stx, %g1, %o1 + %o3)) -+ EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 - 73: andcc %o2, 0x8, %g0 - be,pt %XCC, 1f - nop -- EX_LD(LOAD(ldx, %o1, %o5)) -+ EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) - sub %o2, 0x8, %o2 -- EX_ST(STORE(stx, %o5, %o1 + %o3)) -+ EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) - add %o1, 0x8, %o1 - 1: andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop -- EX_LD(LOAD(lduw, %o1, %o5)) -+ EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) - sub %o2, 0x4, %o2 -- EX_ST(STORE(stw, %o5, %o1 + %o3)) -+ EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) - add %o1, 0x4, %o1 - 1: cmp %o2, 0 - be,pt %XCC, 85f -@@ -503,9 +616,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %g0, %g1, %g1 - sub %o2, %g1, %o2 - --1: EX_LD(LOAD(ldub, %o1, %o5)) -+1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) - subcc %g1, 1, %g1 -- EX_ST(STORE(stb, %o5, %o1 + %o3)) -+ EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) - bgu,pt %icc, 1b - add %o1, 1, %o1 - -@@ -521,16 +634,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 8: mov 64, %o3 - andn %o1, 0x7, %o1 -- EX_LD(LOAD(ldx, %o1, %g2)) -+ EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) - sub %o3, %g1, %o3 - andn %o2, 0x7, %GLOBAL_SPARE - sllx %g2, %g1, %g2 --1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) -+1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) - subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE - add %o1, 0x8, %o1 - srlx %g3, %o3, %o5 - or %o5, %g2, %o5 -- EX_ST(STORE(stx, %o5, %o0)) -+ EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 -@@ -548,9 +661,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - bne,pn %XCC, 90f - sub %o0, %o1, %o3 - --1: EX_LD(LOAD(lduw, %o1, %g1)) -+1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) - subcc %o2, 4, %o2 -- EX_ST(STORE(stw, %g1, %o1 + %o3)) -+ EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -@@ -558,9 +671,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - mov EX_RETVAL(%o4), %o0 - - .align 32 --90: EX_LD(LOAD(ldub, %o1, %g1)) -+90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) - subcc %o2, 1, %o2 -- EX_ST(STORE(stb, %g1, %o1 + %o3)) -+ EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl -diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S -index 88ad73d..db73010 100644 ---- a/arch/sparc/lib/U3copy_from_user.S -+++ b/arch/sparc/lib/U3copy_from_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) - */ - --#define EX_LD(x) \ -+#define EX_LD(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_LD_FP(x) \ -+#define EX_LD_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S -index 845139d..c4ee858 100644 ---- a/arch/sparc/lib/U3copy_to_user.S -+++ b/arch/sparc/lib/U3copy_to_user.S -@@ -3,19 +3,19 @@ - * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) - */ - --#define EX_ST(x) \ -+#define EX_ST(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, y; \ - .text; \ - .align 4; - --#define EX_ST_FP(x) \ -+#define EX_ST_FP(x,y) \ - 98: x; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one_fp;\ -+ .word 98b, y##_fp; \ - .text; \ - .align 4; - -diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S -index 491ee69..54f9870 100644 ---- a/arch/sparc/lib/U3memcpy.S -+++ b/arch/sparc/lib/U3memcpy.S -@@ -4,6 +4,7 @@ - */ - - #ifdef __KERNEL__ -+#include <linux/linkage.h> - #include <asm/visasm.h> - #include <asm/asi.h> - #define GLOBAL_SPARE %g7 -@@ -22,21 +23,17 @@ - #endif - - #ifndef EX_LD --#define EX_LD(x) x -+#define EX_LD(x,y) x - #endif - #ifndef EX_LD_FP --#define EX_LD_FP(x) x -+#define EX_LD_FP(x,y) x - #endif - - #ifndef EX_ST --#define EX_ST(x) x -+#define EX_ST(x,y) x - #endif - #ifndef EX_ST_FP --#define EX_ST_FP(x) x --#endif -- --#ifndef EX_RETVAL --#define EX_RETVAL(x) x -+#define EX_ST_FP(x,y) x - #endif - - #ifndef LOAD -@@ -77,6 +74,87 @@ - */ - - .text -+#ifndef EX_RETVAL -+#define EX_RETVAL(x) x -+__restore_fp: -+ VISExitHalf -+ retl -+ nop -+ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) -+ add %g1, 1, %g1 -+ add %g2, %g1, %g2 -+ ba,pt %xcc, __restore_fp -+ add %o2, %g2, %o0 -+ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) -+ENTRY(U3_retl_o2_plus_g2_fp) -+ ba,pt %xcc, __restore_fp -+ add %o2, %g2, %o0 -+ENDPROC(U3_retl_o2_plus_g2_fp) -+ENTRY(U3_retl_o2_plus_g2_plus_8_fp) -+ add %g2, 8, %g2 -+ ba,pt %xcc, __restore_fp -+ add %o2, %g2, %o0 -+ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) -+ENTRY(U3_retl_o2) -+ retl -+ mov %o2, %o0 -+ENDPROC(U3_retl_o2) -+ENTRY(U3_retl_o2_plus_1) -+ retl -+ add %o2, 1, %o0 -+ENDPROC(U3_retl_o2_plus_1) -+ENTRY(U3_retl_o2_plus_4) -+ retl -+ add %o2, 4, %o0 -+ENDPROC(U3_retl_o2_plus_4) -+ENTRY(U3_retl_o2_plus_8) -+ retl -+ add %o2, 8, %o0 -+ENDPROC(U3_retl_o2_plus_8) -+ENTRY(U3_retl_o2_plus_g1_plus_1) -+ add %g1, 1, %g1 -+ retl -+ add %o2, %g1, %o0 -+ENDPROC(U3_retl_o2_plus_g1_plus_1) -+ENTRY(U3_retl_o2_fp) -+ ba,pt %xcc, __restore_fp -+ mov %o2, %o0 -+ENDPROC(U3_retl_o2_fp) -+ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) -+ sll %o3, 6, %o3 -+ add %o3, 0x80, %o3 -+ ba,pt %xcc, __restore_fp -+ add %o2, %o3, %o0 -+ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) -+ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) -+ sll %o3, 6, %o3 -+ add %o3, 0x40, %o3 -+ ba,pt %xcc, __restore_fp -+ add %o2, %o3, %o0 -+ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) -+ENTRY(U3_retl_o2_plus_GS_plus_0x10) -+ add GLOBAL_SPARE, 0x10, GLOBAL_SPARE -+ retl -+ add %o2, GLOBAL_SPARE, %o0 -+ENDPROC(U3_retl_o2_plus_GS_plus_0x10) -+ENTRY(U3_retl_o2_plus_GS_plus_0x08) -+ add GLOBAL_SPARE, 0x08, GLOBAL_SPARE -+ retl -+ add %o2, GLOBAL_SPARE, %o0 -+ENDPROC(U3_retl_o2_plus_GS_plus_0x08) -+ENTRY(U3_retl_o2_and_7_plus_GS) -+ and %o2, 7, %o2 -+ retl -+ add %o2, GLOBAL_SPARE, %o2 -+ENDPROC(U3_retl_o2_and_7_plus_GS) -+ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) -+ add GLOBAL_SPARE, 8, GLOBAL_SPARE -+ and %o2, 7, %o2 -+ retl -+ add %o2, GLOBAL_SPARE, %o2 -+ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) -+#endif -+ - .align 64 - - /* The cheetah's flexible spine, oversized liver, enlarged heart, -@@ -126,8 +204,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - and %g2, 0x38, %g2 - - 1: subcc %g1, 0x1, %g1 -- EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3)) -- EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE)) -+ EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) -+ EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) - bgu,pt %XCC, 1b - add %o1, 0x1, %o1 - -@@ -138,20 +216,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - be,pt %icc, 3f - alignaddr %o1, %g0, %o1 - -- EX_LD_FP(LOAD(ldd, %o1, %f4)) --1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6)) -+ EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) -+1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f4, %f6, %f0 -- EX_ST_FP(STORE(std, %f0, %o0)) -+ EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) - be,pn %icc, 3f - add %o0, 0x8, %o0 - -- EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f6, %f4, %f2 -- EX_ST_FP(STORE(std, %f2, %o0)) -+ EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) - bne,pt %icc, 1b - add %o0, 0x8, %o0 - -@@ -161,25 +239,25 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - LOAD(prefetch, %o1 + 0x080, #one_read) - LOAD(prefetch, %o1 + 0x0c0, #one_read) - LOAD(prefetch, %o1 + 0x100, #one_read) -- EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) - LOAD(prefetch, %o1 + 0x140, #one_read) -- EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) - LOAD(prefetch, %o1 + 0x180, #one_read) -- EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) - LOAD(prefetch, %o1 + 0x1c0, #one_read) - faligndata %f0, %f2, %f16 -- EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) - faligndata %f2, %f4, %f18 -- EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) - faligndata %f4, %f6, %f20 -- EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) - faligndata %f6, %f8, %f22 - -- EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) - faligndata %f8, %f10, %f24 -- EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) - faligndata %f10, %f12, %f26 -- EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) - - subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE - add %o1, 0x40, %o1 -@@ -190,26 +268,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - .align 64 - 1: -- EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) - faligndata %f12, %f14, %f28 -- EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) - faligndata %f14, %f0, %f30 -- EX_ST_FP(STORE_BLK(%f16, %o0)) -- EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) -+ EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f0, %f2, %f16 - add %o0, 0x40, %o0 - -- EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f2, %f4, %f18 -- EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f4, %f6, %f20 -- EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) - subcc %o3, 0x01, %o3 - faligndata %f6, %f8, %f22 -- EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) - - faligndata %f8, %f10, %f24 -- EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) - LOAD(prefetch, %o1 + 0x1c0, #one_read) - faligndata %f10, %f12, %f26 - bg,pt %XCC, 1b -@@ -217,29 +295,29 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - /* Finally we copy the last full 64-byte block. */ - 2: -- EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) - faligndata %f12, %f14, %f28 -- EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) - faligndata %f14, %f0, %f30 -- EX_ST_FP(STORE_BLK(%f16, %o0)) -- EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6)) -+ EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f0, %f2, %f16 -- EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f2, %f4, %f18 -- EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f4, %f6, %f20 -- EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f6, %f8, %f22 -- EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) - faligndata %f8, %f10, %f24 - cmp %g1, 0 - be,pt %XCC, 1f - add %o0, 0x40, %o0 -- EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) - 1: faligndata %f10, %f12, %f26 - faligndata %f12, %f14, %f28 - faligndata %f14, %f0, %f30 -- EX_ST_FP(STORE_BLK(%f16, %o0)) -+ EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) - add %o0, 0x40, %o0 - add %o1, 0x40, %o1 - membar #Sync -@@ -259,20 +337,20 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - sub %o2, %g2, %o2 - be,a,pt %XCC, 1f -- EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) - --1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2)) -+1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f0, %f2, %f8 -- EX_ST_FP(STORE(std, %f8, %o0)) -+ EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) - be,pn %XCC, 2f - add %o0, 0x8, %o0 -- EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0)) -+ EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) - add %o1, 0x8, %o1 - subcc %g2, 0x8, %g2 - faligndata %f2, %f0, %f8 -- EX_ST_FP(STORE(std, %f8, %o0)) -+ EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) - bne,pn %XCC, 1b - add %o0, 0x8, %o0 - -@@ -292,30 +370,33 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andcc %o2, 0x8, %g0 - be,pt %icc, 1f - nop -- EX_LD(LOAD(ldx, %o1, %o5)) -- EX_ST(STORE(stx, %o5, %o1 + %o3)) -+ EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) -+ EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) - add %o1, 0x8, %o1 -+ sub %o2, 8, %o2 - - 1: andcc %o2, 0x4, %g0 - be,pt %icc, 1f - nop -- EX_LD(LOAD(lduw, %o1, %o5)) -- EX_ST(STORE(stw, %o5, %o1 + %o3)) -+ EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) -+ EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) - add %o1, 0x4, %o1 -+ sub %o2, 4, %o2 - - 1: andcc %o2, 0x2, %g0 - be,pt %icc, 1f - nop -- EX_LD(LOAD(lduh, %o1, %o5)) -- EX_ST(STORE(sth, %o5, %o1 + %o3)) -+ EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) -+ EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) - add %o1, 0x2, %o1 -+ sub %o2, 2, %o2 - - 1: andcc %o2, 0x1, %g0 - be,pt %icc, 85f - nop -- EX_LD(LOAD(ldub, %o1, %o5)) -+ EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) - ba,pt %xcc, 85f -- EX_ST(STORE(stb, %o5, %o1 + %o3)) -+ EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) - - .align 64 - 70: /* 16 < len <= 64 */ -@@ -326,26 +407,26 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - andn %o2, 0xf, GLOBAL_SPARE - and %o2, 0xf, %o2 - 1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE -- EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) -- EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) -- EX_ST(STORE(stx, %o5, %o1 + %o3)) -+ EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) -+ EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) -+ EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) - add %o1, 0x8, %o1 -- EX_ST(STORE(stx, %g1, %o1 + %o3)) -+ EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) - bgu,pt %XCC, 1b - add %o1, 0x8, %o1 - 73: andcc %o2, 0x8, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x8, %o2 -- EX_LD(LOAD(ldx, %o1, %o5)) -- EX_ST(STORE(stx, %o5, %o1 + %o3)) -+ EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) -+ EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) - add %o1, 0x8, %o1 - 1: andcc %o2, 0x4, %g0 - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 -- EX_LD(LOAD(lduw, %o1, %o5)) -- EX_ST(STORE(stw, %o5, %o1 + %o3)) -+ EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) -+ EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) - add %o1, 0x4, %o1 - 1: cmp %o2, 0 - be,pt %XCC, 85f -@@ -361,8 +442,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - sub %o2, %g1, %o2 - - 1: subcc %g1, 1, %g1 -- EX_LD(LOAD(ldub, %o1, %o5)) -- EX_ST(STORE(stb, %o5, %o1 + %o3)) -+ EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) -+ EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) - bgu,pt %icc, 1b - add %o1, 1, %o1 - -@@ -378,16 +459,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 8: mov 64, %o3 - andn %o1, 0x7, %o1 -- EX_LD(LOAD(ldx, %o1, %g2)) -+ EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) - sub %o3, %g1, %o3 - andn %o2, 0x7, GLOBAL_SPARE - sllx %g2, %g1, %g2 --1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) -+1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) - subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE - add %o1, 0x8, %o1 - srlx %g3, %o3, %o5 - or %o5, %g2, %o5 -- EX_ST(STORE(stx, %o5, %o0)) -+ EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) - add %o0, 0x8, %o0 - bgu,pt %icc, 1b - sllx %g3, %g1, %g2 -@@ -407,8 +488,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - - 1: - subcc %o2, 4, %o2 -- EX_LD(LOAD(lduw, %o1, %g1)) -- EX_ST(STORE(stw, %g1, %o1 + %o3)) -+ EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) -+ EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) - bgu,pt %XCC, 1b - add %o1, 4, %o1 - -@@ -418,8 +499,8 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ - .align 32 - 90: - subcc %o2, 1, %o2 -- EX_LD(LOAD(ldub, %o1, %g1)) -- EX_ST(STORE(stb, %g1, %o1 + %o3)) -+ EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) -+ EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) - bgu,pt %XCC, 90b - add %o1, 1, %o1 - retl -diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S -index 302c0e6..4c89b48 100644 ---- a/arch/sparc/lib/copy_in_user.S -+++ b/arch/sparc/lib/copy_in_user.S -@@ -8,18 +8,33 @@ - - #define XCC xcc - --#define EX(x,y) \ -+#define EX(x,y,z) \ - 98: x,y; \ - .section __ex_table,"a";\ - .align 4; \ -- .word 98b, __retl_one; \ -+ .word 98b, z; \ - .text; \ - .align 4; - -+#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) -+#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) -+#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) -+ - .register %g2,#scratch - .register %g3,#scratch - - .text -+__retl_o4_plus_8: -+ add %o4, %o2, %o4 -+ retl -+ add %o4, 8, %o0 -+__retl_o2_plus_4: -+ retl -+ add %o2, 4, %o0 -+__retl_o2_plus_1: -+ retl -+ add %o2, 1, %o0 -+ - .align 32 - - /* Don't try to get too fancy here, just nice and -@@ -44,8 +59,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ - andn %o2, 0x7, %o4 - and %o2, 0x7, %o2 - 1: subcc %o4, 0x8, %o4 -- EX(ldxa [%o1] %asi, %o5) -- EX(stxa %o5, [%o0] %asi) -+ EX_O4(ldxa [%o1] %asi, %o5) -+ EX_O4(stxa %o5, [%o0] %asi) - add %o1, 0x8, %o1 - bgu,pt %XCC, 1b - add %o0, 0x8, %o0 -@@ -53,8 +68,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ - be,pt %XCC, 1f - nop - sub %o2, 0x4, %o2 -- EX(lduwa [%o1] %asi, %o5) -- EX(stwa %o5, [%o0] %asi) -+ EX_O2_4(lduwa [%o1] %asi, %o5) -+ EX_O2_4(stwa %o5, [%o0] %asi) - add %o1, 0x4, %o1 - add %o0, 0x4, %o0 - 1: cmp %o2, 0 -@@ -70,8 +85,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ - - 82: - subcc %o2, 4, %o2 -- EX(lduwa [%o1] %asi, %g1) -- EX(stwa %g1, [%o0] %asi) -+ EX_O2_4(lduwa [%o1] %asi, %g1) -+ EX_O2_4(stwa %g1, [%o0] %asi) - add %o1, 4, %o1 - bgu,pt %XCC, 82b - add %o0, 4, %o0 -@@ -82,8 +97,8 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ - .align 32 - 90: - subcc %o2, 1, %o2 -- EX(lduba [%o1] %asi, %g1) -- EX(stba %g1, [%o0] %asi) -+ EX_O2_1(lduba [%o1] %asi, %g1) -+ EX_O2_1(stba %g1, [%o0] %asi) - add %o1, 1, %o1 - bgu,pt %XCC, 90b - add %o0, 1, %o0 -diff --git a/arch/sparc/lib/user_fixup.c b/arch/sparc/lib/user_fixup.c -deleted file mode 100644 -index ac96ae2..0000000 ---- a/arch/sparc/lib/user_fixup.c -+++ /dev/null -@@ -1,71 +0,0 @@ --/* user_fixup.c: Fix up user copy faults. -- * -- * Copyright (C) 2004 David S. Miller <davem@redhat.com> -- */ -- --#include <linux/compiler.h> --#include <linux/kernel.h> --#include <linux/string.h> --#include <linux/errno.h> --#include <linux/module.h> -- --#include <asm/uaccess.h> -- --/* Calculating the exact fault address when using -- * block loads and stores can be very complicated. -- * -- * Instead of trying to be clever and handling all -- * of the cases, just fix things up simply here. -- */ -- --static unsigned long compute_size(unsigned long start, unsigned long size, unsigned long *offset) --{ -- unsigned long fault_addr = current_thread_info()->fault_address; -- unsigned long end = start + size; -- -- if (fault_addr < start || fault_addr >= end) { -- *offset = 0; -- } else { -- *offset = fault_addr - start; -- size = end - fault_addr; -- } -- return size; --} -- --unsigned long copy_from_user_fixup(void *to, const void __user *from, unsigned long size) --{ -- unsigned long offset; -- -- size = compute_size((unsigned long) from, size, &offset); -- if (likely(size)) -- memset(to + offset, 0, size); -- -- return size; --} --EXPORT_SYMBOL(copy_from_user_fixup); -- --unsigned long copy_to_user_fixup(void __user *to, const void *from, unsigned long size) --{ -- unsigned long offset; -- -- return compute_size((unsigned long) to, size, &offset); --} --EXPORT_SYMBOL(copy_to_user_fixup); -- --unsigned long copy_in_user_fixup(void __user *to, void __user *from, unsigned long size) --{ -- unsigned long fault_addr = current_thread_info()->fault_address; -- unsigned long start = (unsigned long) to; -- unsigned long end = start + size; -- -- if (fault_addr >= start && fault_addr < end) -- return end - fault_addr; -- -- start = (unsigned long) from; -- end = start + size; -- if (fault_addr >= start && fault_addr < end) -- return end - fault_addr; -- -- return size; --} --EXPORT_SYMBOL(copy_in_user_fixup); -diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c -index f2b7711..e20fbba 100644 ---- a/arch/sparc/mm/tsb.c -+++ b/arch/sparc/mm/tsb.c -@@ -27,6 +27,20 @@ static inline int tag_compare(unsigned long tag, unsigned long vaddr) - return (tag == (vaddr >> 22)); - } - -+static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) -+{ -+ unsigned long idx; -+ -+ for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { -+ struct tsb *ent = &swapper_tsb[idx]; -+ unsigned long match = idx << 13; -+ -+ match |= (ent->tag << 22); -+ if (match >= start && match < end) -+ ent->tag = (1UL << TSB_TAG_INVALID_BIT); -+ } -+} -+ - /* TSB flushes need only occur on the processor initiating the address - * space modification, not on each cpu the address space has run on. - * Only the TLB flush needs that treatment. -@@ -36,6 +50,9 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) - { - unsigned long v; - -+ if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) -+ return flush_tsb_kernel_range_scan(start, end); -+ - for (v = start; v < end; v += PAGE_SIZE) { - unsigned long hash = tsb_hash(v, PAGE_SHIFT, - KERNEL_TSB_NENTRIES); -diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S -index b4f4733..5d2fd6c 100644 ---- a/arch/sparc/mm/ultra.S -+++ b/arch/sparc/mm/ultra.S -@@ -30,7 +30,7 @@ - .text - .align 32 - .globl __flush_tlb_mm --__flush_tlb_mm: /* 18 insns */ -+__flush_tlb_mm: /* 19 insns */ - /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ - ldxa [%o1] ASI_DMMU, %g2 - cmp %g2, %o0 -@@ -81,7 +81,7 @@ __flush_tlb_page: /* 22 insns */ - - .align 32 - .globl __flush_tlb_pending --__flush_tlb_pending: /* 26 insns */ -+__flush_tlb_pending: /* 27 insns */ - /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ - rdpr %pstate, %g7 - sllx %o1, 3, %o1 -@@ -113,12 +113,14 @@ __flush_tlb_pending: /* 26 insns */ - - .align 32 - .globl __flush_tlb_kernel_range --__flush_tlb_kernel_range: /* 16 insns */ -+__flush_tlb_kernel_range: /* 31 insns */ - /* %o0=start, %o1=end */ - cmp %o0, %o1 - be,pn %xcc, 2f -+ sub %o1, %o0, %o3 -+ srlx %o3, 18, %o4 -+ brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow - sethi %hi(PAGE_SIZE), %o4 -- sub %o1, %o0, %o3 - sub %o3, %o4, %o3 - or %o0, 0x20, %o0 ! Nucleus - 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP -@@ -131,6 +133,41 @@ __flush_tlb_kernel_range: /* 16 insns */ - retl - nop - nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ -+__spitfire_flush_tlb_kernel_range_slow: -+ mov 63 * 8, %o4 -+1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 -+ andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ -+ bne,pn %xcc, 2f -+ mov TLB_TAG_ACCESS, %o3 -+ stxa %g0, [%o3] ASI_IMMU -+ stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS -+ membar #Sync -+2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 -+ andcc %o3, 0x40, %g0 -+ bne,pn %xcc, 2f -+ mov TLB_TAG_ACCESS, %o3 -+ stxa %g0, [%o3] ASI_DMMU -+ stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS -+ membar #Sync -+2: sub %o4, 8, %o4 -+ brgez,pt %o4, 1b -+ nop -+ retl -+ nop - - __spitfire_flush_tlb_mm_slow: - rdpr %pstate, %g1 -@@ -285,6 +322,40 @@ __cheetah_flush_tlb_pending: /* 27 insns */ - retl - wrpr %g7, 0x0, %pstate - -+__cheetah_flush_tlb_kernel_range: /* 31 insns */ -+ /* %o0=start, %o1=end */ -+ cmp %o0, %o1 -+ be,pn %xcc, 2f -+ sub %o1, %o0, %o3 -+ srlx %o3, 18, %o4 -+ brnz,pn %o4, 3f -+ sethi %hi(PAGE_SIZE), %o4 -+ sub %o3, %o4, %o3 -+ or %o0, 0x20, %o0 ! Nucleus -+1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP -+ stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP -+ membar #Sync -+ brnz,pt %o3, 1b -+ sub %o3, %o4, %o3 -+2: sethi %hi(KERNBASE), %o3 -+ flush %o3 -+ retl -+ nop -+3: mov 0x80, %o4 -+ stxa %g0, [%o4] ASI_DMMU_DEMAP -+ membar #Sync -+ stxa %g0, [%o4] ASI_IMMU_DEMAP -+ membar #Sync -+ retl -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ - #ifdef DCACHE_ALIASING_POSSIBLE - __cheetah_flush_dcache_page: /* 11 insns */ - sethi %hi(PAGE_OFFSET), %g1 -@@ -309,19 +380,28 @@ __hypervisor_tlb_tl0_error: - ret - restore - --__hypervisor_flush_tlb_mm: /* 10 insns */ -+__hypervisor_flush_tlb_mm: /* 19 insns */ - mov %o0, %o2 /* ARG2: mmu context */ - mov 0, %o0 /* ARG0: CPU lists unimplemented */ - mov 0, %o1 /* ARG1: CPU lists unimplemented */ - mov HV_MMU_ALL, %o3 /* ARG3: flags */ - mov HV_FAST_MMU_DEMAP_CTX, %o5 - ta HV_FAST_TRAP -- brnz,pn %o0, __hypervisor_tlb_tl0_error -+ brnz,pn %o0, 1f - mov HV_FAST_MMU_DEMAP_CTX, %o1 - retl - nop -+1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 -+ jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop - --__hypervisor_flush_tlb_page: /* 11 insns */ -+__hypervisor_flush_tlb_page: /* 22 insns */ - /* %o0 = context, %o1 = vaddr */ - mov %o0, %g2 - mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ -@@ -330,12 +410,23 @@ __hypervisor_flush_tlb_page: /* 11 insns */ - srlx %o0, PAGE_SHIFT, %o0 - sllx %o0, PAGE_SHIFT, %o0 - ta HV_MMU_UNMAP_ADDR_TRAP -- brnz,pn %o0, __hypervisor_tlb_tl0_error -+ brnz,pn %o0, 1f - mov HV_MMU_UNMAP_ADDR_TRAP, %o1 - retl - nop -+1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 -+ jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop - --__hypervisor_flush_tlb_pending: /* 16 insns */ -+__hypervisor_flush_tlb_pending: /* 27 insns */ - /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ - sllx %o1, 3, %g1 - mov %o2, %g2 -@@ -347,31 +438,57 @@ __hypervisor_flush_tlb_pending: /* 16 insns */ - srlx %o0, PAGE_SHIFT, %o0 - sllx %o0, PAGE_SHIFT, %o0 - ta HV_MMU_UNMAP_ADDR_TRAP -- brnz,pn %o0, __hypervisor_tlb_tl0_error -+ brnz,pn %o0, 1f - mov HV_MMU_UNMAP_ADDR_TRAP, %o1 - brnz,pt %g1, 1b - nop - retl - nop -+1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 -+ jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop - --__hypervisor_flush_tlb_kernel_range: /* 16 insns */ -+__hypervisor_flush_tlb_kernel_range: /* 31 insns */ - /* %o0=start, %o1=end */ - cmp %o0, %o1 - be,pn %xcc, 2f -- sethi %hi(PAGE_SIZE), %g3 -- mov %o0, %g1 -- sub %o1, %g1, %g2 -+ sub %o1, %o0, %g2 -+ srlx %g2, 18, %g3 -+ brnz,pn %g3, 4f -+ mov %o0, %g1 -+ sethi %hi(PAGE_SIZE), %g3 - sub %g2, %g3, %g2 - 1: add %g1, %g2, %o0 /* ARG0: virtual address */ - mov 0, %o1 /* ARG1: mmu context */ - mov HV_MMU_ALL, %o2 /* ARG2: flags */ - ta HV_MMU_UNMAP_ADDR_TRAP -- brnz,pn %o0, __hypervisor_tlb_tl0_error -+ brnz,pn %o0, 3f - mov HV_MMU_UNMAP_ADDR_TRAP, %o1 - brnz,pt %g2, 1b - sub %g2, %g3, %g2 - 2: retl - nop -+3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 -+ jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 -+ nop -+4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ -+ mov 0, %o1 /* ARG1: CPU lists unimplemented */ -+ mov 0, %o2 /* ARG2: mmu context == nucleus */ -+ mov HV_MMU_ALL, %o3 /* ARG3: flags */ -+ mov HV_FAST_MMU_DEMAP_CTX, %o5 -+ ta HV_FAST_TRAP -+ brnz,pn %o0, 3b -+ mov HV_FAST_MMU_DEMAP_CTX, %o1 -+ retl -+ nop - - #ifdef DCACHE_ALIASING_POSSIBLE - /* XXX Niagara and friends have an 8K cache, so no aliasing is -@@ -394,43 +511,6 @@ tlb_patch_one: - retl - nop - -- .globl cheetah_patch_cachetlbops --cheetah_patch_cachetlbops: -- save %sp, -128, %sp -- -- sethi %hi(__flush_tlb_mm), %o0 -- or %o0, %lo(__flush_tlb_mm), %o0 -- sethi %hi(__cheetah_flush_tlb_mm), %o1 -- or %o1, %lo(__cheetah_flush_tlb_mm), %o1 -- call tlb_patch_one -- mov 19, %o2 -- -- sethi %hi(__flush_tlb_page), %o0 -- or %o0, %lo(__flush_tlb_page), %o0 -- sethi %hi(__cheetah_flush_tlb_page), %o1 -- or %o1, %lo(__cheetah_flush_tlb_page), %o1 -- call tlb_patch_one -- mov 22, %o2 -- -- sethi %hi(__flush_tlb_pending), %o0 -- or %o0, %lo(__flush_tlb_pending), %o0 -- sethi %hi(__cheetah_flush_tlb_pending), %o1 -- or %o1, %lo(__cheetah_flush_tlb_pending), %o1 -- call tlb_patch_one -- mov 27, %o2 -- --#ifdef DCACHE_ALIASING_POSSIBLE -- sethi %hi(__flush_dcache_page), %o0 -- or %o0, %lo(__flush_dcache_page), %o0 -- sethi %hi(__cheetah_flush_dcache_page), %o1 -- or %o1, %lo(__cheetah_flush_dcache_page), %o1 -- call tlb_patch_one -- mov 11, %o2 --#endif /* DCACHE_ALIASING_POSSIBLE */ -- -- ret -- restore -- - #ifdef CONFIG_SMP - /* These are all called by the slaves of a cross call, at - * trap level 1, with interrupts fully disabled. -@@ -447,7 +527,7 @@ cheetah_patch_cachetlbops: - */ - .align 32 - .globl xcall_flush_tlb_mm --xcall_flush_tlb_mm: /* 21 insns */ -+xcall_flush_tlb_mm: /* 24 insns */ - mov PRIMARY_CONTEXT, %g2 - ldxa [%g2] ASI_DMMU, %g3 - srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 -@@ -469,9 +549,12 @@ xcall_flush_tlb_mm: /* 21 insns */ - nop - nop - nop -+ nop -+ nop -+ nop - - .globl xcall_flush_tlb_page --xcall_flush_tlb_page: /* 17 insns */ -+xcall_flush_tlb_page: /* 20 insns */ - /* %g5=context, %g1=vaddr */ - mov PRIMARY_CONTEXT, %g4 - ldxa [%g4] ASI_DMMU, %g2 -@@ -490,15 +573,20 @@ xcall_flush_tlb_page: /* 17 insns */ - retry - nop - nop -+ nop -+ nop -+ nop - - .globl xcall_flush_tlb_kernel_range --xcall_flush_tlb_kernel_range: /* 25 insns */ -+xcall_flush_tlb_kernel_range: /* 44 insns */ - sethi %hi(PAGE_SIZE - 1), %g2 - or %g2, %lo(PAGE_SIZE - 1), %g2 - andn %g1, %g2, %g1 - andn %g7, %g2, %g7 - sub %g7, %g1, %g3 -- add %g2, 1, %g2 -+ srlx %g3, 18, %g2 -+ brnz,pn %g2, 2f -+ add %g2, 1, %g2 - sub %g3, %g2, %g3 - or %g1, 0x20, %g1 ! Nucleus - 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP -@@ -507,8 +595,25 @@ xcall_flush_tlb_kernel_range: /* 25 insns */ - brnz,pt %g3, 1b - sub %g3, %g2, %g3 - retry -- nop -- nop -+2: mov 63 * 8, %g1 -+1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 -+ andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ -+ bne,pn %xcc, 2f -+ mov TLB_TAG_ACCESS, %g2 -+ stxa %g0, [%g2] ASI_IMMU -+ stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS -+ membar #Sync -+2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 -+ andcc %g2, 0x40, %g0 -+ bne,pn %xcc, 2f -+ mov TLB_TAG_ACCESS, %g2 -+ stxa %g0, [%g2] ASI_DMMU -+ stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS -+ membar #Sync -+2: sub %g1, 8, %g1 -+ brgez,pt %g1, 1b -+ nop -+ retry - nop - nop - nop -@@ -637,6 +742,52 @@ xcall_fetch_glob_pmu_n4: - - retry - -+__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ -+ sethi %hi(PAGE_SIZE - 1), %g2 -+ or %g2, %lo(PAGE_SIZE - 1), %g2 -+ andn %g1, %g2, %g1 -+ andn %g7, %g2, %g7 -+ sub %g7, %g1, %g3 -+ srlx %g3, 18, %g2 -+ brnz,pn %g2, 2f -+ add %g2, 1, %g2 -+ sub %g3, %g2, %g3 -+ or %g1, 0x20, %g1 ! Nucleus -+1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP -+ stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP -+ membar #Sync -+ brnz,pt %g3, 1b -+ sub %g3, %g2, %g3 -+ retry -+2: mov 0x80, %g2 -+ stxa %g0, [%g2] ASI_DMMU_DEMAP -+ membar #Sync -+ stxa %g0, [%g2] ASI_IMMU_DEMAP -+ membar #Sync -+ retry -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ nop -+ - #ifdef DCACHE_ALIASING_POSSIBLE - .align 32 - .globl xcall_flush_dcache_page_cheetah -@@ -700,7 +851,7 @@ __hypervisor_tlb_xcall_error: - ba,a,pt %xcc, rtrap - - .globl __hypervisor_xcall_flush_tlb_mm --__hypervisor_xcall_flush_tlb_mm: /* 21 insns */ -+__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ - /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ - mov %o0, %g2 - mov %o1, %g3 -@@ -714,7 +865,7 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ - mov HV_FAST_MMU_DEMAP_CTX, %o5 - ta HV_FAST_TRAP - mov HV_FAST_MMU_DEMAP_CTX, %g6 -- brnz,pn %o0, __hypervisor_tlb_xcall_error -+ brnz,pn %o0, 1f - mov %o0, %g5 - mov %g2, %o0 - mov %g3, %o1 -@@ -723,9 +874,12 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ - mov %g7, %o5 - membar #Sync - retry -+1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 -+ jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 -+ nop - - .globl __hypervisor_xcall_flush_tlb_page --__hypervisor_xcall_flush_tlb_page: /* 17 insns */ -+__hypervisor_xcall_flush_tlb_page: /* 20 insns */ - /* %g5=ctx, %g1=vaddr */ - mov %o0, %g2 - mov %o1, %g3 -@@ -737,42 +891,64 @@ __hypervisor_xcall_flush_tlb_page: /* 17 insns */ - sllx %o0, PAGE_SHIFT, %o0 - ta HV_MMU_UNMAP_ADDR_TRAP - mov HV_MMU_UNMAP_ADDR_TRAP, %g6 -- brnz,a,pn %o0, __hypervisor_tlb_xcall_error -+ brnz,a,pn %o0, 1f - mov %o0, %g5 - mov %g2, %o0 - mov %g3, %o1 - mov %g4, %o2 - membar #Sync - retry -+1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 -+ jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 -+ nop - - .globl __hypervisor_xcall_flush_tlb_kernel_range --__hypervisor_xcall_flush_tlb_kernel_range: /* 25 insns */ -+__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ - /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ - sethi %hi(PAGE_SIZE - 1), %g2 - or %g2, %lo(PAGE_SIZE - 1), %g2 - andn %g1, %g2, %g1 - andn %g7, %g2, %g7 - sub %g7, %g1, %g3 -+ srlx %g3, 18, %g7 - add %g2, 1, %g2 - sub %g3, %g2, %g3 - mov %o0, %g2 - mov %o1, %g4 -- mov %o2, %g7 -+ brnz,pn %g7, 2f -+ mov %o2, %g7 - 1: add %g1, %g3, %o0 /* ARG0: virtual address */ - mov 0, %o1 /* ARG1: mmu context */ - mov HV_MMU_ALL, %o2 /* ARG2: flags */ - ta HV_MMU_UNMAP_ADDR_TRAP - mov HV_MMU_UNMAP_ADDR_TRAP, %g6 -- brnz,pn %o0, __hypervisor_tlb_xcall_error -+ brnz,pn %o0, 1f - mov %o0, %g5 - sethi %hi(PAGE_SIZE), %o2 - brnz,pt %g3, 1b - sub %g3, %o2, %g3 -- mov %g2, %o0 -+5: mov %g2, %o0 - mov %g4, %o1 - mov %g7, %o2 - membar #Sync - retry -+1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 -+ jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 -+ nop -+2: mov %o3, %g1 -+ mov %o5, %g3 -+ mov 0, %o0 /* ARG0: CPU lists unimplemented */ -+ mov 0, %o1 /* ARG1: CPU lists unimplemented */ -+ mov 0, %o2 /* ARG2: mmu context == nucleus */ -+ mov HV_MMU_ALL, %o3 /* ARG3: flags */ -+ mov HV_FAST_MMU_DEMAP_CTX, %o5 -+ ta HV_FAST_TRAP -+ mov %g1, %o3 -+ brz,pt %o0, 5b -+ mov %g3, %o5 -+ mov HV_FAST_MMU_DEMAP_CTX, %g6 -+ ba,pt %xcc, 1b -+ clr %g5 - - /* These just get rescheduled to PIL vectors. */ - .globl xcall_call_function -@@ -809,6 +985,58 @@ xcall_kgdb_capture: - - #endif /* CONFIG_SMP */ - -+ .globl cheetah_patch_cachetlbops -+cheetah_patch_cachetlbops: -+ save %sp, -128, %sp -+ -+ sethi %hi(__flush_tlb_mm), %o0 -+ or %o0, %lo(__flush_tlb_mm), %o0 -+ sethi %hi(__cheetah_flush_tlb_mm), %o1 -+ or %o1, %lo(__cheetah_flush_tlb_mm), %o1 -+ call tlb_patch_one -+ mov 19, %o2 -+ -+ sethi %hi(__flush_tlb_page), %o0 -+ or %o0, %lo(__flush_tlb_page), %o0 -+ sethi %hi(__cheetah_flush_tlb_page), %o1 -+ or %o1, %lo(__cheetah_flush_tlb_page), %o1 -+ call tlb_patch_one -+ mov 22, %o2 -+ -+ sethi %hi(__flush_tlb_pending), %o0 -+ or %o0, %lo(__flush_tlb_pending), %o0 -+ sethi %hi(__cheetah_flush_tlb_pending), %o1 -+ or %o1, %lo(__cheetah_flush_tlb_pending), %o1 -+ call tlb_patch_one -+ mov 27, %o2 -+ -+ sethi %hi(__flush_tlb_kernel_range), %o0 -+ or %o0, %lo(__flush_tlb_kernel_range), %o0 -+ sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 -+ or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 -+ call tlb_patch_one -+ mov 31, %o2 -+ -+#ifdef DCACHE_ALIASING_POSSIBLE -+ sethi %hi(__flush_dcache_page), %o0 -+ or %o0, %lo(__flush_dcache_page), %o0 -+ sethi %hi(__cheetah_flush_dcache_page), %o1 -+ or %o1, %lo(__cheetah_flush_dcache_page), %o1 -+ call tlb_patch_one -+ mov 11, %o2 -+#endif /* DCACHE_ALIASING_POSSIBLE */ -+ -+#ifdef CONFIG_SMP -+ sethi %hi(xcall_flush_tlb_kernel_range), %o0 -+ or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 -+ sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 -+ or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 -+ call tlb_patch_one -+ mov 44, %o2 -+#endif /* CONFIG_SMP */ -+ -+ ret -+ restore - - .globl hypervisor_patch_cachetlbops - hypervisor_patch_cachetlbops: -@@ -819,28 +1047,28 @@ hypervisor_patch_cachetlbops: - sethi %hi(__hypervisor_flush_tlb_mm), %o1 - or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 - call tlb_patch_one -- mov 10, %o2 -+ mov 19, %o2 - - sethi %hi(__flush_tlb_page), %o0 - or %o0, %lo(__flush_tlb_page), %o0 - sethi %hi(__hypervisor_flush_tlb_page), %o1 - or %o1, %lo(__hypervisor_flush_tlb_page), %o1 - call tlb_patch_one -- mov 11, %o2 -+ mov 22, %o2 - - sethi %hi(__flush_tlb_pending), %o0 - or %o0, %lo(__flush_tlb_pending), %o0 - sethi %hi(__hypervisor_flush_tlb_pending), %o1 - or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 - call tlb_patch_one -- mov 16, %o2 -+ mov 27, %o2 - - sethi %hi(__flush_tlb_kernel_range), %o0 - or %o0, %lo(__flush_tlb_kernel_range), %o0 - sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 - or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 - call tlb_patch_one -- mov 16, %o2 -+ mov 31, %o2 - - #ifdef DCACHE_ALIASING_POSSIBLE - sethi %hi(__flush_dcache_page), %o0 -@@ -857,21 +1085,21 @@ hypervisor_patch_cachetlbops: - sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 - or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 - call tlb_patch_one -- mov 21, %o2 -+ mov 24, %o2 - - sethi %hi(xcall_flush_tlb_page), %o0 - or %o0, %lo(xcall_flush_tlb_page), %o0 - sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 - or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 - call tlb_patch_one -- mov 17, %o2 -+ mov 20, %o2 - - sethi %hi(xcall_flush_tlb_kernel_range), %o0 - or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 - sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 - or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 - call tlb_patch_one -- mov 25, %o2 -+ mov 44, %o2 - #endif /* CONFIG_SMP */ - - ret -diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c -index c4751ec..45e87c9 100644 ---- a/drivers/net/ethernet/broadcom/bgmac.c -+++ b/drivers/net/ethernet/broadcom/bgmac.c -@@ -307,6 +307,10 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac, - u32 ctl; - - ctl = bgmac_read(bgmac, ring->mmio_base + BGMAC_DMA_RX_CTL); -+ -+ /* preserve ONLY bits 16-17 from current hardware value */ -+ ctl &= BGMAC_DMA_RX_ADDREXT_MASK; -+ - if (bgmac->feature_flags & BGMAC_FEAT_RX_MASK_SETUP) { - ctl &= ~BGMAC_DMA_RX_BL_MASK; - ctl |= BGMAC_DMA_RX_BL_128 << BGMAC_DMA_RX_BL_SHIFT; -@@ -317,7 +321,6 @@ static void bgmac_dma_rx_enable(struct bgmac *bgmac, - ctl &= ~BGMAC_DMA_RX_PT_MASK; - ctl |= BGMAC_DMA_RX_PT_1 << BGMAC_DMA_RX_PT_SHIFT; - } -- ctl &= BGMAC_DMA_RX_ADDREXT_MASK; - ctl |= BGMAC_DMA_RX_ENABLE; - ctl |= BGMAC_DMA_RX_PARITY_DISABLE; - ctl |= BGMAC_DMA_RX_OVERFLOW_CONT; -diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c -index 505ceaf..2c850a9 100644 ---- a/drivers/net/ethernet/broadcom/bnx2.c -+++ b/drivers/net/ethernet/broadcom/bnx2.c -@@ -49,6 +49,7 @@ - #include <linux/firmware.h> - #include <linux/log2.h> - #include <linux/aer.h> -+#include <linux/crash_dump.h> - - #if defined(CONFIG_CNIC) || defined(CONFIG_CNIC_MODULE) - #define BCM_CNIC 1 -@@ -4759,15 +4760,16 @@ bnx2_setup_msix_tbl(struct bnx2 *bp) - BNX2_WR(bp, BNX2_PCI_GRC_WINDOW3_ADDR, BNX2_MSIX_PBA_ADDR); - } - --static int --bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) -+static void -+bnx2_wait_dma_complete(struct bnx2 *bp) - { - u32 val; -- int i, rc = 0; -- u8 old_port; -+ int i; - -- /* Wait for the current PCI transaction to complete before -- * issuing a reset. */ -+ /* -+ * Wait for the current PCI transaction to complete before -+ * issuing a reset. -+ */ - if ((BNX2_CHIP(bp) == BNX2_CHIP_5706) || - (BNX2_CHIP(bp) == BNX2_CHIP_5708)) { - BNX2_WR(bp, BNX2_MISC_ENABLE_CLR_BITS, -@@ -4791,6 +4793,21 @@ bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) - } - } - -+ return; -+} -+ -+ -+static int -+bnx2_reset_chip(struct bnx2 *bp, u32 reset_code) -+{ -+ u32 val; -+ int i, rc = 0; -+ u8 old_port; -+ -+ /* Wait for the current PCI transaction to complete before -+ * issuing a reset. */ -+ bnx2_wait_dma_complete(bp); -+ - /* Wait for the firmware to tell us it is ok to issue a reset. */ - bnx2_fw_sync(bp, BNX2_DRV_MSG_DATA_WAIT0 | reset_code, 1, 1); - -@@ -6356,6 +6373,10 @@ bnx2_open(struct net_device *dev) - struct bnx2 *bp = netdev_priv(dev); - int rc; - -+ rc = bnx2_request_firmware(bp); -+ if (rc < 0) -+ goto out; -+ - netif_carrier_off(dev); - - bnx2_disable_int(bp); -@@ -6424,6 +6445,7 @@ bnx2_open(struct net_device *dev) - bnx2_free_irq(bp); - bnx2_free_mem(bp); - bnx2_del_napi(bp); -+ bnx2_release_firmware(bp); - goto out; - } - -@@ -8570,12 +8592,15 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) - - pci_set_drvdata(pdev, dev); - -- rc = bnx2_request_firmware(bp); -- if (rc < 0) -- goto error; -- -+ /* -+ * In-flight DMA from 1st kernel could continue going in kdump kernel. -+ * New io-page table has been created before bnx2 does reset at open stage. -+ * We have to wait for the in-flight DMA to complete to avoid it look up -+ * into the newly created io-page table. -+ */ -+ if (is_kdump_kernel()) -+ bnx2_wait_dma_complete(bp); - -- bnx2_reset_chip(bp, BNX2_DRV_MSG_CODE_RESET); - memcpy(dev->dev_addr, bp->mac_addr, ETH_ALEN); - - dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG | -@@ -8608,7 +8633,6 @@ bnx2_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) - return 0; - - error: -- bnx2_release_firmware(bp); - pci_iounmap(pdev, bp->regview); - pci_release_regions(pdev); - pci_disable_device(pdev); -diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c -index d48873b..5cdc96b 100644 ---- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c -+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c -@@ -231,7 +231,7 @@ mlxsw_sp_span_entry_create(struct mlxsw_sp_port *port) - - span_entry->used = true; - span_entry->id = index; -- span_entry->ref_count = 0; -+ span_entry->ref_count = 1; - span_entry->local_port = local_port; - return span_entry; - } -@@ -268,6 +268,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port) - - span_entry = mlxsw_sp_span_entry_find(port); - if (span_entry) { -+ /* Already exists, just take a reference */ - span_entry->ref_count++; - return span_entry; - } -@@ -278,6 +279,7 @@ struct mlxsw_sp_span_entry *mlxsw_sp_span_entry_get(struct mlxsw_sp_port *port) - static int mlxsw_sp_span_entry_put(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_span_entry *span_entry) - { -+ WARN_ON(!span_entry->ref_count); - if (--span_entry->ref_count == 0) - mlxsw_sp_span_entry_destroy(mlxsw_sp, span_entry); - return 0; -diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c -index 3f5c51d..62514b9 100644 ---- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c -+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c -@@ -777,6 +777,26 @@ static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp, - } - } - -+static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl) -+{ -+ u8 num_rec, last_rec_index, num_entries; -+ -+ num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl); -+ last_rec_index = num_rec - 1; -+ -+ if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM) -+ return false; -+ if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) == -+ MLXSW_REG_RAUHTD_TYPE_IPV6) -+ return true; -+ -+ num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl, -+ last_rec_index); -+ if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC) -+ return true; -+ return false; -+} -+ - static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) - { - char *rauhtd_pl; -@@ -803,7 +823,7 @@ static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp) - for (i = 0; i < num_rec; i++) - mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl, - i); -- } while (num_rec); -+ } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl)); - rtnl_unlock(); - - kfree(rauhtd_pl); -diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c -index 4c8c60a..fe9e7b1 100644 ---- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c -+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c -@@ -871,6 +871,13 @@ static int stmmac_init_phy(struct net_device *dev) - return -ENODEV; - } - -+ /* stmmac_adjust_link will change this to PHY_IGNORE_INTERRUPT to avoid -+ * subsequent PHY polling, make sure we force a link transition if -+ * we have a UP/DOWN/UP transition -+ */ -+ if (phydev->is_pseudo_fixed_link) -+ phydev->irq = PHY_POLL; -+ - pr_debug("stmmac_init_phy: %s: attached to PHY (UID 0x%x)" - " Link = %d\n", dev->name, phydev->phy_id, phydev->link); - -diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c -index 5c8429f..3a5530d 100644 ---- a/drivers/usb/gadget/function/f_fs.c -+++ b/drivers/usb/gadget/function/f_fs.c -@@ -133,8 +133,60 @@ struct ffs_epfile { - /* - * Buffer for holding data from partial reads which may happen since - * we’re rounding user read requests to a multiple of a max packet size. -+ * -+ * The pointer is initialised with NULL value and may be set by -+ * __ffs_epfile_read_data function to point to a temporary buffer. -+ * -+ * In normal operation, calls to __ffs_epfile_read_buffered will consume -+ * data from said buffer and eventually free it. Importantly, while the -+ * function is using the buffer, it sets the pointer to NULL. This is -+ * all right since __ffs_epfile_read_data and __ffs_epfile_read_buffered -+ * can never run concurrently (they are synchronised by epfile->mutex) -+ * so the latter will not assign a new value to the pointer. -+ * -+ * Meanwhile ffs_func_eps_disable frees the buffer (if the pointer is -+ * valid) and sets the pointer to READ_BUFFER_DROP value. This special -+ * value is crux of the synchronisation between ffs_func_eps_disable and -+ * __ffs_epfile_read_data. -+ * -+ * Once __ffs_epfile_read_data is about to finish it will try to set the -+ * pointer back to its old value (as described above), but seeing as the -+ * pointer is not-NULL (namely READ_BUFFER_DROP) it will instead free -+ * the buffer. -+ * -+ * == State transitions == -+ * -+ * • ptr == NULL: (initial state) -+ * ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP -+ * ◦ __ffs_epfile_read_buffered: nop -+ * ◦ __ffs_epfile_read_data allocates temp buffer: go to ptr == buf -+ * ◦ reading finishes: n/a, not in ‘and reading’ state -+ * • ptr == DROP: -+ * ◦ __ffs_epfile_read_buffer_free: nop -+ * ◦ __ffs_epfile_read_buffered: go to ptr == NULL -+ * ◦ __ffs_epfile_read_data allocates temp buffer: free buf, nop -+ * ◦ reading finishes: n/a, not in ‘and reading’ state -+ * • ptr == buf: -+ * ◦ __ffs_epfile_read_buffer_free: free buf, go to ptr == DROP -+ * ◦ __ffs_epfile_read_buffered: go to ptr == NULL and reading -+ * ◦ __ffs_epfile_read_data: n/a, __ffs_epfile_read_buffered -+ * is always called first -+ * ◦ reading finishes: n/a, not in ‘and reading’ state -+ * • ptr == NULL and reading: -+ * ◦ __ffs_epfile_read_buffer_free: go to ptr == DROP and reading -+ * ◦ __ffs_epfile_read_buffered: n/a, mutex is held -+ * ◦ __ffs_epfile_read_data: n/a, mutex is held -+ * ◦ reading finishes and … -+ * … all data read: free buf, go to ptr == NULL -+ * … otherwise: go to ptr == buf and reading -+ * • ptr == DROP and reading: -+ * ◦ __ffs_epfile_read_buffer_free: nop -+ * ◦ __ffs_epfile_read_buffered: n/a, mutex is held -+ * ◦ __ffs_epfile_read_data: n/a, mutex is held -+ * ◦ reading finishes: free buf, go to ptr == DROP - */ -- struct ffs_buffer *read_buffer; /* P: epfile->mutex */ -+ struct ffs_buffer *read_buffer; -+#define READ_BUFFER_DROP ((struct ffs_buffer *)ERR_PTR(-ESHUTDOWN)) - - char name[5]; - -@@ -733,25 +785,47 @@ static void ffs_epfile_async_io_complete(struct usb_ep *_ep, - schedule_work(&io_data->work); - } - -+static void __ffs_epfile_read_buffer_free(struct ffs_epfile *epfile) -+{ -+ /* -+ * See comment in struct ffs_epfile for full read_buffer pointer -+ * synchronisation story. -+ */ -+ struct ffs_buffer *buf = xchg(&epfile->read_buffer, READ_BUFFER_DROP); -+ if (buf && buf != READ_BUFFER_DROP) -+ kfree(buf); -+} -+ - /* Assumes epfile->mutex is held. */ - static ssize_t __ffs_epfile_read_buffered(struct ffs_epfile *epfile, - struct iov_iter *iter) - { -- struct ffs_buffer *buf = epfile->read_buffer; -+ /* -+ * Null out epfile->read_buffer so ffs_func_eps_disable does not free -+ * the buffer while we are using it. See comment in struct ffs_epfile -+ * for full read_buffer pointer synchronisation story. -+ */ -+ struct ffs_buffer *buf = xchg(&epfile->read_buffer, NULL); - ssize_t ret; -- if (!buf) -+ if (!buf || buf == READ_BUFFER_DROP) - return 0; - - ret = copy_to_iter(buf->data, buf->length, iter); - if (buf->length == ret) { - kfree(buf); -- epfile->read_buffer = NULL; -- } else if (unlikely(iov_iter_count(iter))) { -+ return ret; -+ } -+ -+ if (unlikely(iov_iter_count(iter))) { - ret = -EFAULT; - } else { - buf->length -= ret; - buf->data += ret; - } -+ -+ if (cmpxchg(&epfile->read_buffer, NULL, buf)) -+ kfree(buf); -+ - return ret; - } - -@@ -780,7 +854,15 @@ static ssize_t __ffs_epfile_read_data(struct ffs_epfile *epfile, - buf->length = data_len; - buf->data = buf->storage; - memcpy(buf->storage, data + ret, data_len); -- epfile->read_buffer = buf; -+ -+ /* -+ * At this point read_buffer is NULL or READ_BUFFER_DROP (if -+ * ffs_func_eps_disable has been called in the meanwhile). See comment -+ * in struct ffs_epfile for full read_buffer pointer synchronisation -+ * story. -+ */ -+ if (unlikely(cmpxchg(&epfile->read_buffer, NULL, buf))) -+ kfree(buf); - - return ret; - } -@@ -1094,8 +1176,7 @@ ffs_epfile_release(struct inode *inode, struct file *file) - - ENTER(); - -- kfree(epfile->read_buffer); -- epfile->read_buffer = NULL; -+ __ffs_epfile_read_buffer_free(epfile); - ffs_data_closed(epfile->ffs); - - return 0; -@@ -1721,24 +1802,20 @@ static void ffs_func_eps_disable(struct ffs_function *func) - unsigned count = func->ffs->eps_count; - unsigned long flags; - -+ spin_lock_irqsave(&func->ffs->eps_lock, flags); - do { -- if (epfile) -- mutex_lock(&epfile->mutex); -- spin_lock_irqsave(&func->ffs->eps_lock, flags); - /* pending requests get nuked */ - if (likely(ep->ep)) - usb_ep_disable(ep->ep); - ++ep; -- spin_unlock_irqrestore(&func->ffs->eps_lock, flags); - - if (epfile) { - epfile->ep = NULL; -- kfree(epfile->read_buffer); -- epfile->read_buffer = NULL; -- mutex_unlock(&epfile->mutex); -+ __ffs_epfile_read_buffer_free(epfile); - ++epfile; - } - } while (--count); -+ spin_unlock_irqrestore(&func->ffs->eps_lock, flags); - } - - static int ffs_func_eps_enable(struct ffs_function *func) -diff --git a/include/net/ip.h b/include/net/ip.h -index 156b0c1..0ccf6da 100644 ---- a/include/net/ip.h -+++ b/include/net/ip.h -@@ -47,7 +47,6 @@ struct inet_skb_parm { - #define IPSKB_REROUTED BIT(4) - #define IPSKB_DOREDIRECT BIT(5) - #define IPSKB_FRAG_PMTU BIT(6) --#define IPSKB_FRAG_SEGS BIT(7) - - u16 frag_max_size; - }; -diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h -index 43a5a0e..b01d5d1 100644 ---- a/include/net/ip6_tunnel.h -+++ b/include/net/ip6_tunnel.h -@@ -145,6 +145,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, - { - int pkt_len, err; - -+ memset(skb->cb, 0, sizeof(struct inet6_skb_parm)); - pkt_len = skb->len - skb_inner_network_offset(skb); - err = ip6_local_out(dev_net(skb_dst(skb)->dev), sk, skb); - if (unlikely(net_xmit_eval(err))) -diff --git a/include/net/sock.h b/include/net/sock.h -index 8741988..c26eab9 100644 ---- a/include/net/sock.h -+++ b/include/net/sock.h -@@ -1587,11 +1587,11 @@ static inline void sock_put(struct sock *sk) - void sock_gen_put(struct sock *sk); - - int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested, -- unsigned int trim_cap); -+ unsigned int trim_cap, bool refcounted); - static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb, - const int nested) - { -- return __sk_receive_skb(sk, skb, nested, 1); -+ return __sk_receive_skb(sk, skb, nested, 1, true); - } - - static inline void sk_tx_queue_set(struct sock *sk, int tx_queue) -diff --git a/include/net/tcp.h b/include/net/tcp.h -index 7717302..0de6989 100644 ---- a/include/net/tcp.h -+++ b/include/net/tcp.h -@@ -1164,6 +1164,7 @@ static inline void tcp_prequeue_init(struct tcp_sock *tp) - } - - bool tcp_prequeue(struct sock *sk, struct sk_buff *skb); -+int tcp_filter(struct sock *sk, struct sk_buff *skb); - - #undef STATE_TRACE - -diff --git a/include/uapi/linux/atm_zatm.h b/include/uapi/linux/atm_zatm.h -index 5cd4d4d..9c9c6ad 100644 ---- a/include/uapi/linux/atm_zatm.h -+++ b/include/uapi/linux/atm_zatm.h -@@ -14,7 +14,6 @@ - - #include <linux/atmapi.h> - #include <linux/atmioc.h> --#include <linux/time.h> - - #define ZATM_GETPOOL _IOW('a',ATMIOC_SARPRV+1,struct atmif_sioc) - /* get pool statistics */ -diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c -index 570eeca..ad1bc67 100644 ---- a/kernel/bpf/hashtab.c -+++ b/kernel/bpf/hashtab.c -@@ -687,7 +687,8 @@ static void delete_all_elements(struct bpf_htab *htab) - - hlist_for_each_entry_safe(l, n, head, hash_node) { - hlist_del_rcu(&l->hash_node); -- htab_elem_free(htab, l); -+ if (l->state != HTAB_EXTRA_ELEM_USED) -+ htab_elem_free(htab, l); - } - } - } -diff --git a/net/core/dev.c b/net/core/dev.c -index 44b3ba4..9ce9d72 100644 ---- a/net/core/dev.c -+++ b/net/core/dev.c -@@ -2484,7 +2484,7 @@ int skb_checksum_help(struct sk_buff *skb) - goto out; - } - -- *(__sum16 *)(skb->data + offset) = csum_fold(csum); -+ *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; - out_set_summed: - skb->ip_summed = CHECKSUM_NONE; - out: -diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c -index 52742a0..5550a86 100644 ---- a/net/core/flow_dissector.c -+++ b/net/core/flow_dissector.c -@@ -118,7 +118,7 @@ bool __skb_flow_dissect(const struct sk_buff *skb, - struct flow_dissector_key_tags *key_tags; - struct flow_dissector_key_keyid *key_keyid; - u8 ip_proto = 0; -- bool ret = false; -+ bool ret; - - if (!data) { - data = skb->data; -@@ -481,12 +481,17 @@ bool __skb_flow_dissect(const struct sk_buff *skb, - out_good: - ret = true; - --out_bad: -+ key_control->thoff = (u16)nhoff; -+out: - key_basic->n_proto = proto; - key_basic->ip_proto = ip_proto; -- key_control->thoff = (u16)nhoff; - - return ret; -+ -+out_bad: -+ ret = false; -+ key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); -+ goto out; - } - EXPORT_SYMBOL(__skb_flow_dissect); - -diff --git a/net/core/sock.c b/net/core/sock.c -index fd7b41e..10acacc 100644 ---- a/net/core/sock.c -+++ b/net/core/sock.c -@@ -453,7 +453,7 @@ int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) - EXPORT_SYMBOL(sock_queue_rcv_skb); - - int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, -- const int nested, unsigned int trim_cap) -+ const int nested, unsigned int trim_cap, bool refcounted) - { - int rc = NET_RX_SUCCESS; - -@@ -487,7 +487,8 @@ int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, - - bh_unlock_sock(sk); - out: -- sock_put(sk); -+ if (refcounted) -+ sock_put(sk); - return rc; - discard_and_relse: - kfree_skb(skb); -@@ -1563,6 +1564,7 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) - RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); - - newsk->sk_err = 0; -+ newsk->sk_err_soft = 0; - newsk->sk_priority = 0; - newsk->sk_incoming_cpu = raw_smp_processor_id(); - atomic64_set(&newsk->sk_cookie, 0); -diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c -index 345a3ae..b567c87 100644 ---- a/net/dccp/ipv4.c -+++ b/net/dccp/ipv4.c -@@ -235,7 +235,7 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info) - { - const struct iphdr *iph = (struct iphdr *)skb->data; - const u8 offset = iph->ihl << 2; -- const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset); -+ const struct dccp_hdr *dh; - struct dccp_sock *dp; - struct inet_sock *inet; - const int type = icmp_hdr(skb)->type; -@@ -245,11 +245,13 @@ static void dccp_v4_err(struct sk_buff *skb, u32 info) - int err; - struct net *net = dev_net(skb->dev); - -- if (skb->len < offset + sizeof(*dh) || -- skb->len < offset + __dccp_basic_hdr_len(dh)) { -- __ICMP_INC_STATS(net, ICMP_MIB_INERRORS); -- return; -- } -+ /* Only need dccph_dport & dccph_sport which are the first -+ * 4 bytes in dccp header. -+ * Our caller (icmp_socket_deliver()) already pulled 8 bytes for us. -+ */ -+ BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8); -+ BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8); -+ dh = (struct dccp_hdr *)(skb->data + offset); - - sk = __inet_lookup_established(net, &dccp_hashinfo, - iph->daddr, dh->dccph_dport, -@@ -868,7 +870,7 @@ static int dccp_v4_rcv(struct sk_buff *skb) - goto discard_and_relse; - nf_reset(skb); - -- return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4); -+ return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, refcounted); - - no_dccp_socket: - if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) -diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c -index 3828f94..715e5d1 100644 ---- a/net/dccp/ipv6.c -+++ b/net/dccp/ipv6.c -@@ -70,7 +70,7 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, - u8 type, u8 code, int offset, __be32 info) - { - const struct ipv6hdr *hdr = (const struct ipv6hdr *)skb->data; -- const struct dccp_hdr *dh = (struct dccp_hdr *)(skb->data + offset); -+ const struct dccp_hdr *dh; - struct dccp_sock *dp; - struct ipv6_pinfo *np; - struct sock *sk; -@@ -78,12 +78,13 @@ static void dccp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, - __u64 seq; - struct net *net = dev_net(skb->dev); - -- if (skb->len < offset + sizeof(*dh) || -- skb->len < offset + __dccp_basic_hdr_len(dh)) { -- __ICMP6_INC_STATS(net, __in6_dev_get(skb->dev), -- ICMP6_MIB_INERRORS); -- return; -- } -+ /* Only need dccph_dport & dccph_sport which are the first -+ * 4 bytes in dccp header. -+ * Our caller (icmpv6_notify()) already pulled 8 bytes for us. -+ */ -+ BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_sport) > 8); -+ BUILD_BUG_ON(offsetofend(struct dccp_hdr, dccph_dport) > 8); -+ dh = (struct dccp_hdr *)(skb->data + offset); - - sk = __inet6_lookup_established(net, &dccp_hashinfo, - &hdr->daddr, dh->dccph_dport, -@@ -738,7 +739,8 @@ static int dccp_v6_rcv(struct sk_buff *skb) - if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) - goto discard_and_relse; - -- return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4) ? -1 : 0; -+ return __sk_receive_skb(sk, skb, 1, dh->dccph_doff * 4, -+ refcounted) ? -1 : 0; - - no_dccp_socket: - if (!xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) -@@ -956,6 +958,7 @@ static const struct inet_connection_sock_af_ops dccp_ipv6_mapped = { - .getsockopt = ipv6_getsockopt, - .addr2sockaddr = inet6_csk_addr2sockaddr, - .sockaddr_len = sizeof(struct sockaddr_in6), -+ .bind_conflict = inet6_csk_bind_conflict, - #ifdef CONFIG_COMPAT - .compat_setsockopt = compat_ipv6_setsockopt, - .compat_getsockopt = compat_ipv6_getsockopt, -diff --git a/net/dccp/proto.c b/net/dccp/proto.c -index 41e6580..9fe25bf 100644 ---- a/net/dccp/proto.c -+++ b/net/dccp/proto.c -@@ -1009,6 +1009,10 @@ void dccp_close(struct sock *sk, long timeout) - __kfree_skb(skb); - } - -+ /* If socket has been already reset kill it. */ -+ if (sk->sk_state == DCCP_CLOSED) -+ goto adjudge_to_death; -+ - if (data_was_unread) { - /* Unread data was tossed, send an appropriate Reset Code */ - DCCP_WARN("ABORT with %u bytes unread\n", data_was_unread); -diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c -index e2ffc2a..7ef7031 100644 ---- a/net/ipv4/fib_trie.c -+++ b/net/ipv4/fib_trie.c -@@ -2455,22 +2455,19 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter, - struct key_vector *l, **tp = &iter->tnode; - t_key key; - -- /* use cache location of next-to-find key */ -+ /* use cached location of previously found key */ - if (iter->pos > 0 && pos >= iter->pos) { -- pos -= iter->pos; - key = iter->key; - } else { -- iter->pos = 0; -+ iter->pos = 1; - key = 0; - } - -- while ((l = leaf_walk_rcu(tp, key)) != NULL) { -+ pos -= iter->pos; -+ -+ while ((l = leaf_walk_rcu(tp, key)) && (pos-- > 0)) { - key = l->key + 1; - iter->pos++; -- -- if (--pos <= 0) -- break; -- - l = NULL; - - /* handle unlikely case of a key wrap */ -@@ -2479,7 +2476,7 @@ static struct key_vector *fib_route_get_idx(struct fib_route_iter *iter, - } - - if (l) -- iter->key = key; /* remember it */ -+ iter->key = l->key; /* remember it */ - else - iter->pos = 0; /* forget it */ - -@@ -2507,7 +2504,7 @@ static void *fib_route_seq_start(struct seq_file *seq, loff_t *pos) - return fib_route_get_idx(iter, *pos); - - iter->pos = 0; -- iter->key = 0; -+ iter->key = KEY_MAX; - - return SEQ_START_TOKEN; - } -@@ -2516,7 +2513,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos) - { - struct fib_route_iter *iter = seq->private; - struct key_vector *l = NULL; -- t_key key = iter->key; -+ t_key key = iter->key + 1; - - ++*pos; - -@@ -2525,7 +2522,7 @@ static void *fib_route_seq_next(struct seq_file *seq, void *v, loff_t *pos) - l = leaf_walk_rcu(&iter->tnode, key); - - if (l) { -- iter->key = l->key + 1; -+ iter->key = l->key; - iter->pos++; - } else { - iter->pos = 0; -diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c -index 38abe70..48734ee 100644 ---- a/net/ipv4/icmp.c -+++ b/net/ipv4/icmp.c -@@ -477,7 +477,7 @@ static struct rtable *icmp_route_lookup(struct net *net, - fl4->flowi4_proto = IPPROTO_ICMP; - fl4->fl4_icmp_type = type; - fl4->fl4_icmp_code = code; -- fl4->flowi4_oif = l3mdev_master_ifindex(skb_in->dev); -+ fl4->flowi4_oif = l3mdev_master_ifindex(skb_dst(skb_in)->dev); - - security_skb_classify_flow(skb_in, flowi4_to_flowi(fl4)); - rt = __ip_route_output_key_hash(net, fl4, -@@ -502,7 +502,7 @@ static struct rtable *icmp_route_lookup(struct net *net, - if (err) - goto relookup_failed; - -- if (inet_addr_type_dev_table(net, skb_in->dev, -+ if (inet_addr_type_dev_table(net, skb_dst(skb_in)->dev, - fl4_dec.saddr) == RTN_LOCAL) { - rt2 = __ip_route_output_key(net, &fl4_dec); - if (IS_ERR(rt2)) -diff --git a/net/ipv4/ip_forward.c b/net/ipv4/ip_forward.c -index 8b4ffd2..9f0a7b9 100644 ---- a/net/ipv4/ip_forward.c -+++ b/net/ipv4/ip_forward.c -@@ -117,7 +117,7 @@ int ip_forward(struct sk_buff *skb) - if (opt->is_strictroute && rt->rt_uses_gateway) - goto sr_failed; - -- IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS; -+ IPCB(skb)->flags |= IPSKB_FORWARDED; - mtu = ip_dst_mtu_maybe_forward(&rt->dst, true); - if (ip_exceeds_mtu(skb, mtu)) { - IP_INC_STATS(net, IPSTATS_MIB_FRAGFAILS); -diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c -index dde37fb..307daed 100644 ---- a/net/ipv4/ip_output.c -+++ b/net/ipv4/ip_output.c -@@ -223,11 +223,9 @@ static int ip_finish_output_gso(struct net *net, struct sock *sk, - struct sk_buff *segs; - int ret = 0; - -- /* common case: fragmentation of segments is not allowed, -- * or seglen is <= mtu -+ /* common case: seglen is <= mtu - */ -- if (((IPCB(skb)->flags & IPSKB_FRAG_SEGS) == 0) || -- skb_gso_validate_mtu(skb, mtu)) -+ if (skb_gso_validate_mtu(skb, mtu)) - return ip_finish_output2(net, sk, skb); - - /* Slowpath - GSO segment length is exceeding the dst MTU. -diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c -index 0f227db..afd6b59 100644 ---- a/net/ipv4/ip_tunnel_core.c -+++ b/net/ipv4/ip_tunnel_core.c -@@ -63,7 +63,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, - int pkt_len = skb->len - skb_inner_network_offset(skb); - struct net *net = dev_net(rt->dst.dev); - struct net_device *dev = skb->dev; -- int skb_iif = skb->skb_iif; - struct iphdr *iph; - int err; - -@@ -73,16 +72,6 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, - skb_dst_set(skb, &rt->dst); - memset(IPCB(skb), 0, sizeof(*IPCB(skb))); - -- if (skb_iif && !(df & htons(IP_DF))) { -- /* Arrived from an ingress interface, got encapsulated, with -- * fragmentation of encapulating frames allowed. -- * If skb is gso, the resulting encapsulated network segments -- * may exceed dst mtu. -- * Allow IP Fragmentation of segments. -- */ -- IPCB(skb)->flags |= IPSKB_FRAG_SEGS; -- } -- - /* Push down and install the IP header. */ - skb_push(skb, sizeof(struct iphdr)); - skb_reset_network_header(skb); -diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c -index 5f006e1..27089f5 100644 ---- a/net/ipv4/ipmr.c -+++ b/net/ipv4/ipmr.c -@@ -1749,7 +1749,7 @@ static void ipmr_queue_xmit(struct net *net, struct mr_table *mrt, - vif->dev->stats.tx_bytes += skb->len; - } - -- IPCB(skb)->flags |= IPSKB_FORWARDED | IPSKB_FRAG_SEGS; -+ IPCB(skb)->flags |= IPSKB_FORWARDED; - - /* RFC1584 teaches, that DVMRP/PIM router must deliver packets locally - * not only before forwarding, but after forwarding on all output -diff --git a/net/ipv4/route.c b/net/ipv4/route.c -index 62c3ed0..2f23ef1 100644 ---- a/net/ipv4/route.c -+++ b/net/ipv4/route.c -@@ -753,7 +753,9 @@ static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flow - goto reject_redirect; - } - -- n = ipv4_neigh_lookup(&rt->dst, NULL, &new_gw); -+ n = __ipv4_neigh_lookup(rt->dst.dev, new_gw); -+ if (!n) -+ n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev); - if (!IS_ERR(n)) { - if (!(n->nud_state & NUD_VALID)) { - neigh_event_send(n, NULL); -diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c -index ffbb218..c876f5d 100644 ---- a/net/ipv4/tcp.c -+++ b/net/ipv4/tcp.c -@@ -1145,7 +1145,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) - - err = -EPIPE; - if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) -- goto out_err; -+ goto do_error; - - sg = !!(sk->sk_route_caps & NETIF_F_SG); - -@@ -1219,7 +1219,7 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) - - if (!skb_can_coalesce(skb, i, pfrag->page, - pfrag->offset)) { -- if (i == sysctl_max_skb_frags || !sg) { -+ if (i >= sysctl_max_skb_frags || !sg) { - tcp_mark_push(tp, skb); - goto new_segment; - } -diff --git a/net/ipv4/tcp_dctcp.c b/net/ipv4/tcp_dctcp.c -index 10d728b..ab37c67 100644 ---- a/net/ipv4/tcp_dctcp.c -+++ b/net/ipv4/tcp_dctcp.c -@@ -56,6 +56,7 @@ struct dctcp { - u32 next_seq; - u32 ce_state; - u32 delayed_ack_reserved; -+ u32 loss_cwnd; - }; - - static unsigned int dctcp_shift_g __read_mostly = 4; /* g = 1/2^4 */ -@@ -96,6 +97,7 @@ static void dctcp_init(struct sock *sk) - ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA); - - ca->delayed_ack_reserved = 0; -+ ca->loss_cwnd = 0; - ca->ce_state = 0; - - dctcp_reset(tp, ca); -@@ -111,9 +113,10 @@ static void dctcp_init(struct sock *sk) - - static u32 dctcp_ssthresh(struct sock *sk) - { -- const struct dctcp *ca = inet_csk_ca(sk); -+ struct dctcp *ca = inet_csk_ca(sk); - struct tcp_sock *tp = tcp_sk(sk); - -+ ca->loss_cwnd = tp->snd_cwnd; - return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U); - } - -@@ -308,12 +311,20 @@ static size_t dctcp_get_info(struct sock *sk, u32 ext, int *attr, - return 0; - } - -+static u32 dctcp_cwnd_undo(struct sock *sk) -+{ -+ const struct dctcp *ca = inet_csk_ca(sk); -+ -+ return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd); -+} -+ - static struct tcp_congestion_ops dctcp __read_mostly = { - .init = dctcp_init, - .in_ack_event = dctcp_update_alpha, - .cwnd_event = dctcp_cwnd_event, - .ssthresh = dctcp_ssthresh, - .cong_avoid = tcp_reno_cong_avoid, -+ .undo_cwnd = dctcp_cwnd_undo, - .set_state = dctcp_state, - .get_info = dctcp_get_info, - .flags = TCP_CONG_NEEDS_ECN, -diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c -index 7158d4f..7b235fa 100644 ---- a/net/ipv4/tcp_ipv4.c -+++ b/net/ipv4/tcp_ipv4.c -@@ -1537,6 +1537,21 @@ bool tcp_prequeue(struct sock *sk, struct sk_buff *skb) - } - EXPORT_SYMBOL(tcp_prequeue); - -+int tcp_filter(struct sock *sk, struct sk_buff *skb) -+{ -+ struct tcphdr *th = (struct tcphdr *)skb->data; -+ unsigned int eaten = skb->len; -+ int err; -+ -+ err = sk_filter_trim_cap(sk, skb, th->doff * 4); -+ if (!err) { -+ eaten -= skb->len; -+ TCP_SKB_CB(skb)->end_seq -= eaten; -+ } -+ return err; -+} -+EXPORT_SYMBOL(tcp_filter); -+ - /* - * From tcp_input.c - */ -@@ -1648,8 +1663,10 @@ int tcp_v4_rcv(struct sk_buff *skb) - - nf_reset(skb); - -- if (sk_filter(sk, skb)) -+ if (tcp_filter(sk, skb)) - goto discard_and_relse; -+ th = (const struct tcphdr *)skb->data; -+ iph = ip_hdr(skb); - - skb->dev = NULL; - -diff --git a/net/ipv6/icmp.c b/net/ipv6/icmp.c -index bd59c34..7370ad2 100644 ---- a/net/ipv6/icmp.c -+++ b/net/ipv6/icmp.c -@@ -448,7 +448,7 @@ static void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, - if (__ipv6_addr_needs_scope_id(addr_type)) - iif = skb->dev->ifindex; - else -- iif = l3mdev_master_ifindex(skb->dev); -+ iif = l3mdev_master_ifindex(skb_dst(skb)->dev); - - /* - * Must not send error if the source does not uniquely -diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c -index fc67822..af6a09e 100644 ---- a/net/ipv6/tcp_ipv6.c -+++ b/net/ipv6/tcp_ipv6.c -@@ -1228,7 +1228,7 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) - if (skb->protocol == htons(ETH_P_IP)) - return tcp_v4_do_rcv(sk, skb); - -- if (sk_filter(sk, skb)) -+ if (tcp_filter(sk, skb)) - goto discard; - - /* -@@ -1455,8 +1455,10 @@ static int tcp_v6_rcv(struct sk_buff *skb) - if (tcp_v6_inbound_md5_hash(sk, skb)) - goto discard_and_relse; - -- if (sk_filter(sk, skb)) -+ if (tcp_filter(sk, skb)) - goto discard_and_relse; -+ th = (const struct tcphdr *)skb->data; -+ hdr = ipv6_hdr(skb); - - skb->dev = NULL; - -diff --git a/net/sctp/socket.c b/net/sctp/socket.c -index baccbf3..7b0e059 100644 ---- a/net/sctp/socket.c -+++ b/net/sctp/socket.c -@@ -1214,9 +1214,12 @@ static int __sctp_connect(struct sock *sk, - - timeo = sock_sndtimeo(sk, f_flags & O_NONBLOCK); - -- err = sctp_wait_for_connect(asoc, &timeo); -- if ((err == 0 || err == -EINPROGRESS) && assoc_id) -+ if (assoc_id) - *assoc_id = asoc->assoc_id; -+ err = sctp_wait_for_connect(asoc, &timeo); -+ /* Note: the asoc may be freed after the return of -+ * sctp_wait_for_connect. -+ */ - - /* Don't free association on exit. */ - asoc = NULL; -@@ -4278,19 +4281,18 @@ static void sctp_shutdown(struct sock *sk, int how) - { - struct net *net = sock_net(sk); - struct sctp_endpoint *ep; -- struct sctp_association *asoc; - - if (!sctp_style(sk, TCP)) - return; - -- if (how & SEND_SHUTDOWN) { -+ ep = sctp_sk(sk)->ep; -+ if (how & SEND_SHUTDOWN && !list_empty(&ep->asocs)) { -+ struct sctp_association *asoc; -+ - sk->sk_state = SCTP_SS_CLOSING; -- ep = sctp_sk(sk)->ep; -- if (!list_empty(&ep->asocs)) { -- asoc = list_entry(ep->asocs.next, -- struct sctp_association, asocs); -- sctp_primitive_SHUTDOWN(net, asoc, NULL); -- } -+ asoc = list_entry(ep->asocs.next, -+ struct sctp_association, asocs); -+ sctp_primitive_SHUTDOWN(net, asoc, NULL); - } - } - -diff --git a/net/socket.c b/net/socket.c -index a1bd161..03bc2c2 100644 ---- a/net/socket.c -+++ b/net/socket.c -@@ -2041,6 +2041,8 @@ int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg, unsigned int vlen, - if (err) - break; - ++datagrams; -+ if (msg_data_left(&msg_sys)) -+ break; - cond_resched(); - } - -diff --git a/tools/spi/spidev_test.c b/tools/spi/spidev_test.c -index f3825b6..f046b77 100644 ---- a/tools/spi/spidev_test.c -+++ b/tools/spi/spidev_test.c -@@ -19,6 +19,7 @@ - #include <getopt.h> - #include <fcntl.h> - #include <sys/ioctl.h> -+#include <linux/ioctl.h> - #include <sys/stat.h> - #include <linux/types.h> - #include <linux/spi/spidev.h> |