x86, 32-bit: improve lazy TLB handling code

Impact: micro-optimize the 32-bit TLB flush code

Use the faster x86_{read,write}_percpu() accessors here.

Signed-off-by: Jan Beulich <jbeulich@novell.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/include/asm/mmu_context_32.h b/arch/x86/include/asm/mmu_context_32.h
index 8e10015781fb2d6794aa0cfabae334956b365c88..7e98ce1d2c0e5ad8b2aca1c1e5d733cd839d9cc8 100644 (file)
--- a/arch/x86/include/asm/mmu_context_32.h
+++ b/arch/x86/include/asm/mmu_context_32.h
@@ -4,9 +4,8 @@
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
 #ifdef CONFIG_SMP
-       unsigned cpu = smp_processor_id();
-       if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
-               per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_LAZY;
+       if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK)
+               x86_write_percpu(cpu_tlbstate.state, TLBSTATE_LAZY);
 #endif
 }
 
@@ -20,8 +19,8 @@ static inline void switch_mm(struct mm_struct *prev,
                /* stop flush ipis for the previous mm */
                cpu_clear(cpu, prev->cpu_vm_mask);
 #ifdef CONFIG_SMP
-               per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
-               per_cpu(cpu_tlbstate, cpu).active_mm = next;
+               x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
+               x86_write_percpu(cpu_tlbstate.active_mm, next);
 #endif
                cpu_set(cpu, next->cpu_vm_mask);
 
@@ -36,8 +35,8 @@ static inline void switch_mm(struct mm_struct *prev,
        }
 #ifdef CONFIG_SMP
        else {
-               per_cpu(cpu_tlbstate, cpu).state = TLBSTATE_OK;
-               BUG_ON(per_cpu(cpu_tlbstate, cpu).active_mm != next);
+               x86_write_percpu(cpu_tlbstate.state, TLBSTATE_OK);
+               BUG_ON(x86_read_percpu(cpu_tlbstate.active_mm) != next);
 
                if (!cpu_test_and_set(cpu, next->cpu_vm_mask)) {
                        /* We were in lazy tlb mode and leave_mm disabled

diff --git a/arch/x86/kernel/tlb_32.c b/arch/x86/kernel/tlb_32.c
index f4049f3513b6b5e01e71523a5490fbedd5db9992..4290d918b58af67f22fa8a25f182d2eebf133819 100644 (file)
--- a/arch/x86/kernel/tlb_32.c
+++ b/arch/x86/kernel/tlb_32.c
@@ -34,9 +34,8 @@ static DEFINE_SPINLOCK(tlbstate_lock);
  */
 void leave_mm(int cpu)
 {
-       if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK)
-               BUG();
-       cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask);
+       BUG_ON(x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK);
+       cpu_clear(cpu, x86_read_percpu(cpu_tlbstate.active_mm)->cpu_vm_mask);
        load_cr3(swapper_pg_dir);
 }
 EXPORT_SYMBOL_GPL(leave_mm);
@@ -104,8 +103,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
                 * BUG();
                 */
 
-       if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) {
-               if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) {
+       if (flush_mm == x86_read_percpu(cpu_tlbstate.active_mm)) {
+               if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_OK) {
                        if (flush_va == TLB_FLUSH_ALL)
                                local_flush_tlb();
                        else
@@ -238,7 +237,7 @@ static void do_flush_tlb_all(void *info)
        unsigned long cpu = smp_processor_id();
 
        __flush_tlb_all();
-       if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY)
+       if (x86_read_percpu(cpu_tlbstate.state) == TLBSTATE_LAZY)
                leave_mm(cpu);
 }