diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 69c4e3d35e020b1d0113899175ff7b3dd1cbe1e2..a21adcb75fc6a16ac0de37e676d0dbe96016143b 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -499,9 +499,9 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
#define __SOFTEN_TEST(h, vec) \
lbz r10,PACASOFTIRQEN(r13); \
- cmpwi r10,IRQS_DISABLED; \
+ andi. r10,r10,IRQS_DISABLED; \
li r10,SOFTEN_VALUE_##vec; \
- beq masked_##h##interrupt
+ bne masked_##h##interrupt
#define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 52afb1595cb0fbbfc46c2add4eacf46fd8a7dde3..f96280ee95402554345dc6bedd44b8f6eaae6783 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -31,8 +31,8 @@
/*
* flags for paca->soft_enabled
*/
-#define IRQS_ENABLED 1
-#define IRQS_DISABLED 0
+#define IRQS_ENABLED 0
+#define IRQS_DISABLED 1
#endif /* CONFIG_PPC64 */
@@ -68,6 +68,18 @@ static inline notrace unsigned long soft_enabled_return(void)
*/
static inline notrace void soft_enabled_set(unsigned long enable)
{
+#ifdef CONFIG_TRACE_IRQFLAGS
+ /*
+ * mask must always include LINUX bit if any are set, and
+ * interrupts don't get replayed until the Linux interrupt is
+ * unmasked. This could be changed to replay partial unmasks
+ * in future, which would allow Linux masks to nest inside
+ * other masks, among other things. For now, be very dumb and
+ * simple.
+ */
+ WARN_ON(mask && !(mask & IRQS_DISABLED));
+#endif
+
asm volatile(
"stb %0,%1(13)"
:
@@ -76,15 +88,19 @@ static inline notrace void soft_enabled_set(unsigned long enable)
: "memory");
}
-static inline notrace unsigned long soft_enabled_set_return(unsigned long enable)
+static inline notrace unsigned long soft_enabled_set_return(unsigned long mask)
{
unsigned long flags;
+#ifdef CONFIG_TRACE_IRQFLAGS
+ WARN_ON(mask && !(mask & IRQS_DISABLED));
+#endif
+
asm volatile(
"lbz %0,%1(13); stb %2,%1(13)"
: "=&r" (flags)
: "i" (offsetof(struct paca_struct, soft_enabled)),
- "r" (enable)
+ "r" (mask)
: "memory");
return flags;
@@ -114,7 +130,7 @@ static inline unsigned long arch_local_irq_save(void)
static inline bool arch_irqs_disabled_flags(unsigned long flags)
{
- return flags == IRQS_DISABLED;
+ return flags & IRQS_DISABLED;
}
static inline bool arch_irqs_disabled(void)
@@ -133,7 +149,7 @@ static inline bool arch_irqs_disabled(void)
#define hard_irq_disable() do { \
unsigned long flags; \
__hard_irq_disable(); \
- flags = soft_enabled_set_return(IRQS_DISABLED); \
+ flags = soft_enabled_set_return(IRQS_DISABLED);\
local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \
if (!arch_irqs_disabled_flags(flags)) \
trace_hardirqs_off(); \
@@ -158,7 +174,7 @@ static inline void may_hard_irq_enable(void)
static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
{
- return (regs->softe == IRQS_DISABLED);
+ return (regs->softe & IRQS_DISABLED);
}
extern bool prep_irq_for_idle(void);
diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h
index 8d9fdc84828d690b2dbd40a50d8cb7496ef6f806..f1ec012232d9412a678969b55a8f8b319652b1b7 100644
--- a/arch/powerpc/include/asm/irqflags.h
+++ b/arch/powerpc/include/asm/irqflags.h
@@ -49,11 +49,11 @@
#define RECONCILE_IRQ_STATE(__rA, __rB) \
lbz __rA,PACASOFTIRQEN(r13); \
lbz __rB,PACAIRQHAPPENED(r13); \
- cmpwi cr0,__rA,IRQS_DISABLED;\
+ andi. __rA,__rA,IRQS_DISABLED;\
li __rA,IRQS_DISABLED; \
ori __rB,__rB,PACA_IRQ_HARD_DIS; \
stb __rB,PACAIRQHAPPENED(r13); \
- beq 44f; \
+ bne 44f; \
stb __rA,PACASOFTIRQEN(r13); \
TRACE_DISABLE_INTS; \
44:
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index f8b32224dc114bc412c89f13d6fedaed978ebc69..51e4cc4dba4af3a7059078d11f7593b8e761c9ce 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -130,8 +130,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
*/
#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
lbz r10,PACASOFTIRQEN(r13)
- xori r10,r10,IRQS_ENABLED
-1: tdnei r10,0
+1: tdnei r10,IRQS_ENABLED
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
#endif
@@ -741,10 +740,10 @@ resume_kernel:
beq+ restore
/* Check that preempt_count() == 0 and interrupts are enabled */
lwz r8,TI_PREEMPT(r9)
- cmpwi cr1,r8,0
+ cmpwi cr0,r8,0
+ bne restore
ld r0,SOFTE(r1)
- cmpdi r0,IRQS_DISABLED
- crandc eq,cr1*4+eq,eq
+ andi. r0,r0,IRQS_DISABLED
bne restore
/*
@@ -783,11 +782,11 @@ restore:
*/
ld r5,SOFTE(r1)
lbz r6,PACASOFTIRQEN(r13)
- cmpwi cr0,r5,IRQS_DISABLED
- beq .Lrestore_irq_off
+ andi. r5,r5,IRQS_DISABLED
+ bne .Lrestore_irq_off
/* We are enabling, were we already enabled ? Yes, just return */
- cmpwi cr0,r6,IRQS_ENABLED
+ andi. r6,r6,IRQS_DISABLED
beq cr0,.Ldo_restore
/*
@@ -1031,15 +1030,15 @@ _GLOBAL(enter_rtas)
li r0,0
mtcr r0
-#ifdef CONFIG_BUG
+#ifdef CONFIG_BUG
/* There is no way it is acceptable to get here with interrupts enabled,
* check it with the asm equivalent of WARN_ON
*/
lbz r0,PACASOFTIRQEN(r13)
-1: tdnei r0,IRQS_DISABLED
+1: tdeqi r0,IRQS_ENABLED
EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
#endif
-
+
/* Hard-disable interrupts */
mfmsr r6
rldicl r7,r6,48,1
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 80cc91bbcf1a4bd7fa3a6db10b5273022c6ed8b3..9216ece7672ce41aa7c9db2543922ea65929e891 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -210,10 +210,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
ld r5,SOFTE(r1)
/* Interrupts had better not already be enabled... */
- twnei r6,IRQS_DISABLED
+ tweqi r6,IRQS_ENABLED
- cmpwi cr0,r5,IRQS_DISABLED
- beq 1f
+ andi. r6,r5,IRQS_DISABLED
+ bne 1f
TRACE_ENABLE_INTS
stb r5,PACASOFTIRQEN(r13)
@@ -352,8 +352,8 @@ ret_from_mc_except:
#define PROLOG_ADDITION_MASKABLE_GEN(n) \
lbz r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \
- cmpwi cr0,r10,IRQS_DISABLED; /* yes -> go out of line */ \
- beq masked_interrupt_book3e_##n
+ andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \
+ bne masked_interrupt_book3e_##n
#define PROLOG_ADDITION_2REGS_GEN(n) \
std r14,PACA_EXGEN+EX_R14(r13); \
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 6c04e465caf51dfc508b299039edda1cfb4e55a8..406c4329b535ec37b839dc4b12f1715210a8f885 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -219,15 +219,29 @@ notrace unsigned int __check_irq_replay(void)
return 0;
}
-notrace void arch_local_irq_restore(unsigned long en)
+notrace void arch_local_irq_restore(unsigned long mask)
{
unsigned char irq_happened;
unsigned int replay;
/* Write the new soft-enabled value */
- soft_enabled_set(en);
- if (en == IRQS_DISABLED)
+ soft_enabled_set(mask);
+ if (mask) {
+#ifdef CONFIG_TRACE_IRQFLAGS
+ /*
+ * mask must always include LINUX bit if any
+ * are set, and interrupts don't get replayed until
+ * the Linux interrupt is unmasked. This could be
+ * changed to replay partial unmasks in future,
+ * which would allow Linux masks to nest inside
+ * other masks, among other things. For now, be very
+ * dumb and simple.
+ */
+ WARN_ON(!(mask & IRQS_DISABLED));
+#endif
return;
+ }
+
/*
* From this point onward, we can take interrupts, preempt,
* etc... unless we got hard-disabled. We check if an event
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 594261e308b3164327b56890c74d817f2ea72a6f..ea4c709f481bc28654591dbb27db453c06a69beb 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -322,7 +322,7 @@ static inline void perf_read_regs(struct pt_regs *regs)
*/
static inline int perf_intr_is_nmi(struct pt_regs *regs)
{
- return (regs->softe == IRQS_DISABLED);
+ return (regs->softe & IRQS_DISABLED);
}
/*