2 * Copyright (C) 2004-2006 Atmel Corporation
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 * This file contains the low-level entry-points into the kernel, that is,
11 * exception handlers, debug trap handlers, interrupt handlers and the
12 * system call handler.
14 #include <linux/errno.h>
17 #include <asm/hardirq.h>
21 #include <asm/pgtable.h>
22 #include <asm/ptrace.h>
23 #include <asm/sysreg.h>
24 #include <asm/thread_info.h>
25 #include <asm/unistd.h>
28 # define preempt_stop mask_interrupts
31 # define fault_resume_kernel fault_restore_all
34 #define __MASK(x) ((1 << (x)) - 1)
35 #define IRQ_MASK ((__MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) | \
36 (__MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT))
38 .section .ex.text,"ax",@progbits
45 bral do_bus_error_write
47 bral do_bus_error_read
51 bral handle_address_fault
53 bral handle_protection_fault
57 bral do_illegal_opcode_ll
59 bral do_illegal_opcode_ll
61 bral do_illegal_opcode_ll
65 bral do_illegal_opcode_ll
67 bral handle_address_fault
69 bral handle_address_fault
71 bral handle_protection_fault
73 bral handle_protection_fault
77 #define tlbmiss_save pushm r0-r3
78 #define tlbmiss_restore popm r0-r3
95 .global tlb_miss_common
98 mfsr r0, SYSREG_TLBEAR
101 /* Is it the vmalloc space? */
103 brcs handle_vmalloc_miss
106 * First level lookup: The PGD contains virtual pointers to
107 * the second-level page tables, but they may be NULL if not
111 lsr r2, r0, PGDIR_SHIFT
113 bfextu r1, r0, PAGE_SHIFT, PGDIR_SHIFT - PAGE_SHIFT
115 breq page_table_not_present
117 /* Second level lookup */
119 mfsr r0, SYSREG_TLBARLO
120 bld r2, _PAGE_BIT_PRESENT
121 brcc page_not_present
123 /* Mark the page as accessed */
124 sbr r2, _PAGE_BIT_ACCESSED
127 /* Drop software flags */
128 andl r2, _PAGE_FLAGS_HARDWARE_MASK & 0xffff
129 mtsr SYSREG_TLBELO, r2
131 /* Figure out which entry we want to replace */
132 mfsr r1, SYSREG_MMUCR
135 mov r3, -1 /* All entries have been accessed, */
136 mov r2, 0 /* so start at 0 */
137 mtsr SYSREG_TLBARLO, r3 /* and reset TLBAR */
139 1: bfins r1, r2, SYSREG_DRP_OFFSET, SYSREG_DRP_SIZE
140 mtsr SYSREG_MMUCR, r1
147 /* Simply do the lookup in init's page table */
148 mov r1, lo(swapper_pg_dir)
149 orh r1, hi(swapper_pg_dir)
152 /* The slow path of the TLB miss handler */
154 page_table_not_present:
159 rcall save_full_context_ex
163 rjmp ret_from_exception
166 /* --- System Call --- */
170 #ifdef CONFIG_PREEMPT
173 pushm r12 /* r12_orig */
176 mfsr r0, SYSREG_RAR_SUP
177 mfsr r1, SYSREG_RSR_SUP
178 #ifdef CONFIG_PREEMPT
184 /* check for syscall tracing */
186 ld.w r1, r0[TI_flags]
187 bld r1, TIF_SYSCALL_TRACE
188 brcs syscall_trace_enter
194 lddpc lr, syscall_table_addr
196 mov r8, r5 /* 5th argument (6th is pushed by stub) */
199 .global syscall_return
202 mask_interrupts /* make sure we don't miss an interrupt
203 setting need_resched or sigpending
204 between sampling and the rets */
206 /* Store the return value so that the correct value is loaded below */
207 stdsp sp[REG_R12], r12
209 ld.w r1, r0[TI_flags]
210 andl r1, _TIF_ALLWORK_MASK, COH
211 brne syscall_exit_work
215 mtsr SYSREG_RAR_SUP, r8
216 mtsr SYSREG_RSR_SUP, r9
218 sub sp, -4 /* r12_orig */
229 .global ret_from_fork
233 /* check for syscall tracing */
235 ld.w r1, r0[TI_flags]
236 andl r1, _TIF_ALLWORK_MASK, COH
237 brne syscall_exit_work
238 rjmp syscall_exit_cont
244 rjmp syscall_trace_cont
247 bld r1, TIF_SYSCALL_TRACE
252 ld.w r1, r0[TI_flags]
254 1: bld r1, TIF_NEED_RESCHED
259 ld.w r1, r0[TI_flags]
262 2: mov r2, _TIF_SIGPENDING | _TIF_RESTORE_SIGMASK
268 rcall do_notify_resume
270 ld.w r1, r0[TI_flags]
273 3: bld r1, TIF_BREAKPOINT
274 brcc syscall_exit_cont
275 rjmp enter_monitor_mode
277 /* This function expects to find offending PC in SYSREG_RAR_EX */
278 .type save_full_context_ex, @function
280 save_full_context_ex:
281 mfsr r11, SYSREG_RAR_EX
282 sub r9, pc, . - debug_trampoline
283 mfsr r8, SYSREG_RSR_EX
287 andh r8, (MODE_MASK >> 16), COH
290 1: pushm r11, r12 /* PC and SR */
294 2: sub r10, sp, -(FRAME_SIZE_FULL - REG_LR)
295 stdsp sp[4], r10 /* replace saved SP */
299 * The debug handler set up a trampoline to make us
300 * automatically enter monitor mode upon return, but since
301 * we're saving the full context, we must assume that the
302 * exception handler might want to alter the return address
303 * and/or status register. So we need to restore the original
304 * context and enter monitor mode manually after the exception
307 3: get_thread_info r8
308 ld.w r11, r8[TI_rar_saved]
309 ld.w r12, r8[TI_rsr_saved]
311 .size save_full_context_ex, . - save_full_context_ex
313 /* Low-level exception handlers */
317 rcall save_full_context_ex
320 rcall do_critical_exception
322 /* We should never get here... */
324 sub r12, pc, (. - 1f)
327 1: .asciz "Return from critical exception!"
333 rcall save_full_context_ex
340 rcall save_full_context_ex
342 1: mfsr r12, SYSREG_BEAR
345 rjmp ret_from_exception
351 mfsr r9, SYSREG_RSR_NMI
352 mfsr r8, SYSREG_RAR_NMI
353 bfextu r0, r9, MODE_SHIFT, 3
356 1: pushm r8, r9 /* PC and SR */
361 mtsr SYSREG_RAR_NMI, r8
363 mtsr SYSREG_RSR_NMI, r9
367 sub sp, -4 /* skip r12_orig */
370 2: sub r10, sp, -(FRAME_SIZE_FULL - REG_LR)
371 stdsp sp[4], r10 /* replace saved SP */
375 sub sp, -4 /* skip sp */
377 sub sp, -4 /* skip r12_orig */
380 handle_address_fault:
383 rcall save_full_context_ex
386 rcall do_address_exception
387 rjmp ret_from_exception
389 handle_protection_fault:
392 rcall save_full_context_ex
396 rjmp ret_from_exception
399 do_illegal_opcode_ll:
402 rcall save_full_context_ex
405 rcall do_illegal_opcode
406 rjmp ret_from_exception
410 mfsr r1, SYSREG_TLBEAR
412 lsr r2, r1, PGDIR_SHIFT
414 lsl r1, (32 - PGDIR_SHIFT)
415 lsr r1, (32 - PGDIR_SHIFT) + PAGE_SHIFT
417 /* Translate to virtual address in P1 */
422 sbr r3, _PAGE_BIT_DIRTY
426 /* The page table is up-to-date. Update the TLB entry as well */
427 andl r0, lo(_PAGE_FLAGS_HARDWARE_MASK)
428 mtsr SYSREG_TLBELO, r0
430 /* MMUCR[DRP] is updated automatically, so let's go... */
439 rcall save_full_context_ex
444 rjmp ret_from_exception
450 andh r4, (MODE_MASK >> 16), COH
451 brne fault_resume_kernel
454 ld.w r1, r0[TI_flags]
455 andl r1, _TIF_WORK_MASK, COH
461 mtsr SYSREG_RAR_EX, r8
462 mtsr SYSREG_RSR_EX, r9
468 #ifdef CONFIG_PREEMPT
470 ld.w r2, r0[TI_preempt_count]
473 ld.w r1, r0[TI_flags]
474 bld r1, TIF_NEED_RESCHED
477 bld r4, SYSREG_GM_OFFSET
479 rcall preempt_schedule_irq
486 mtsr SYSREG_RAR_EX, r8
487 mtsr SYSREG_RSR_EX, r9
489 sub sp, -4 /* ignore SP */
491 sub sp, -4 /* ignore r12_orig */
495 /* Switch to exception mode so that we can share the same code. */
497 cbr r8, SYSREG_M0_OFFSET
498 orh r8, hi(SYSREG_BIT(M1) | SYSREG_BIT(M2))
502 ld.w r1, r0[TI_flags]
505 bld r1, TIF_NEED_RESCHED
510 ld.w r1, r0[TI_flags]
513 1: mov r2, _TIF_SIGPENDING | _TIF_RESTORE_SIGMASK
519 rcall do_notify_resume
521 ld.w r1, r0[TI_flags]
524 2: bld r1, TIF_BREAKPOINT
525 brcc fault_resume_user
526 rjmp enter_monitor_mode
528 .section .kprobes.text, "ax", @progbits
529 .type handle_debug, @function
531 sub sp, 4 /* r12_orig */
533 mfsr r8, SYSREG_RAR_DBG
534 mfsr r9, SYSREG_RSR_DBG
537 bfextu r9, r9, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
538 brne debug_fixup_regs
541 #ifdef CONFIG_TRACE_IRQFLAGS
542 rcall trace_hardirqs_off
549 bfextu r3, r2, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
550 brne debug_resume_kernel
553 ld.w r1, r0[TI_flags]
554 mov r2, _TIF_DBGWORK_MASK
558 bld r1, TIF_SINGLE_STEP
561 sbr r4, OCD_DC_SS_BIT
566 mtsr SYSREG_RSR_DBG, r11
567 mtsr SYSREG_RAR_DBG, r10
568 #ifdef CONFIG_TRACE_IRQFLAGS
569 rcall trace_hardirqs_on
575 .size handle_debug, . - handle_debug
577 /* Mode of the trapped context is in r9 */
578 .type debug_fixup_regs, @function
582 bfins r8, r9, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
588 sub r8, sp, -FRAME_SIZE_FULL
590 rjmp .Ldebug_fixup_cont
591 .size debug_fixup_regs, . - debug_fixup_regs
593 .type debug_resume_kernel, @function
597 mtsr SYSREG_RAR_DBG, r10
598 mtsr SYSREG_RSR_DBG, r11
599 #ifdef CONFIG_TRACE_IRQFLAGS
600 bld r11, SYSREG_GM_OFFSET
602 rcall trace_hardirqs_on
607 bfins r2, r3, SYSREG_MODE_OFFSET, SYSREG_MODE_SIZE
613 sub sp, -4 /* skip SP */
617 .size debug_resume_kernel, . - debug_resume_kernel
619 .type debug_exit_work, @function
622 * We must return from Monitor Mode using a retd, and we must
623 * not schedule since that involves the D bit in SR getting
624 * cleared by something other than the debug hardware. This
625 * may cause undefined behaviour according to the Architecture
628 * So we fix up the return address and status and return to a
629 * stub below in Exception mode. From there, we can follow the
630 * normal exception return path.
632 * The real return address and status registers are stored on
633 * the stack in the way the exception return path understands,
634 * so no need to fix anything up there.
636 sub r8, pc, . - fault_exit_work
637 mtsr SYSREG_RAR_DBG, r8
639 orh r9, hi(SR_EM | SR_GM | MODE_EXCEPTION)
640 mtsr SYSREG_RSR_DBG, r9
643 .size debug_exit_work, . - debug_exit_work
645 .set rsr_int0, SYSREG_RSR_INT0
646 .set rsr_int1, SYSREG_RSR_INT1
647 .set rsr_int2, SYSREG_RSR_INT2
648 .set rsr_int3, SYSREG_RSR_INT3
649 .set rar_int0, SYSREG_RAR_INT0
650 .set rar_int1, SYSREG_RAR_INT1
651 .set rar_int2, SYSREG_RAR_INT2
652 .set rar_int3, SYSREG_RAR_INT3
654 .macro IRQ_LEVEL level
655 .type irq_level\level, @function
657 sub sp, 4 /* r12_orig */
659 mfsr r8, rar_int\level
660 mfsr r9, rsr_int\level
662 #ifdef CONFIG_PREEMPT
663 sub r11, pc, (. - system_call)
676 bfextu r4, r4, SYSREG_M0_OFFSET, 3
677 cp.w r4, MODE_SUPERVISOR >> SYSREG_M0_OFFSET
679 cp.w r4, MODE_USER >> SYSREG_M0_OFFSET
680 #ifdef CONFIG_PREEMPT
687 ld.w r1, r0[TI_flags]
688 andl r1, _TIF_WORK_MASK, COH
692 #ifdef CONFIG_TRACE_IRQFLAGS
693 rcall trace_hardirqs_on
696 mtsr rar_int\level, r8
697 mtsr rsr_int\level, r9
699 sub sp, -4 /* ignore r12_orig */
702 #ifdef CONFIG_PREEMPT
704 mfsr r8, rsr_int\level
706 mtsr rsr_int\level, r8
708 sub sp, -4 /* ignore r12_orig */
712 2: get_thread_info r0
713 ld.w r1, r0[TI_flags]
714 bld r1, TIF_CPU_GOING_TO_SLEEP
715 #ifdef CONFIG_PREEMPT
720 sub r1, pc, . - cpu_idle_skip_sleep
722 #ifdef CONFIG_PREEMPT
723 3: get_thread_info r0
724 ld.w r2, r0[TI_preempt_count]
727 ld.w r1, r0[TI_flags]
728 bld r1, TIF_NEED_RESCHED
731 bld r4, SYSREG_GM_OFFSET
733 rcall preempt_schedule_irq
738 .section .irq.text,"ax",@progbits
749 .section .kprobes.text, "ax", @progbits
750 .type enter_monitor_mode, @function
753 * We need to enter monitor mode to do a single step. The
754 * monitor code will alter the return address so that we
755 * return directly to the user instead of returning here.
758 rjmp breakpoint_failed
760 .size enter_monitor_mode, . - enter_monitor_mode
762 .type debug_trampoline, @function
763 .global debug_trampoline
766 * Save the registers on the stack so that the monitor code
767 * can find them easily.
769 sub sp, 4 /* r12_orig */
772 ld.w r8, r0[TI_rar_saved]
773 ld.w r9, r0[TI_rsr_saved]
777 * The monitor code will alter the return address so we don't
781 rjmp breakpoint_failed
782 .size debug_trampoline, . - debug_trampoline
784 .type breakpoint_failed, @function
787 * Something went wrong. Perhaps the debug hardware isn't
790 lda.w r12, msg_breakpoint_failed
792 mov r10, 9 /* SIGKILL */
796 msg_breakpoint_failed:
797 .asciz "Failed to enter Debug Mode"