2 * Copyright (C) 1995 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * Gareth Hughes <gareth@valinux.com>, May 2000
10 * CPU hotplug support - ashok.raj@intel.com
14 * This file handles the architecture-dependent parts of process handling..
19 #include <linux/cpu.h>
20 #include <linux/errno.h>
21 #include <linux/sched.h>
23 #include <linux/kernel.h>
25 #include <linux/elfcore.h>
26 #include <linux/smp.h>
27 #include <linux/slab.h>
28 #include <linux/user.h>
29 #include <linux/interrupt.h>
30 #include <linux/utsname.h>
31 #include <linux/delay.h>
32 #include <linux/module.h>
33 #include <linux/ptrace.h>
34 #include <linux/random.h>
35 #include <linux/notifier.h>
36 #include <linux/kprobes.h>
37 #include <linux/kdebug.h>
38 #include <linux/tick.h>
39 #include <linux/prctl.h>
40 #include <linux/uaccess.h>
42 #include <linux/ftrace.h>
44 #include <asm/pgtable.h>
45 #include <asm/system.h>
46 #include <asm/processor.h>
48 #include <asm/mmu_context.h>
50 #include <asm/prctl.h>
52 #include <asm/proto.h>
55 #include <asm/syscalls.h>
57 asmlinkage extern void ret_from_fork(void);
59 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
61 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
63 void idle_notifier_register(struct notifier_block *n)
65 atomic_notifier_chain_register(&idle_notifier, n);
67 EXPORT_SYMBOL_GPL(idle_notifier_register);
69 void idle_notifier_unregister(struct notifier_block *n)
71 atomic_notifier_chain_unregister(&idle_notifier, n);
73 EXPORT_SYMBOL_GPL(idle_notifier_unregister);
78 atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
81 static void __exit_idle(void)
83 if (test_and_clear_bit_pda(0, isidle) == 0)
85 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
88 /* Called from interrupts to signify idle end */
91 /* idle loop has pid 0 */
98 static inline void play_dead(void)
105 * The idle thread. There's no useful work to be
106 * done, so just try to conserve power and have a
107 * low exit latency (ie sit in a loop waiting for
108 * somebody to say that they'd like to reschedule)
112 current_thread_info()->status |= TS_POLLING;
113 /* endless idle loop with no priority at all */
115 tick_nohz_stop_sched_tick(1);
116 while (!need_resched()) {
120 if (cpu_is_offline(smp_processor_id()))
123 * Idle routines should keep interrupts disabled
124 * from here on, until they go to idle.
125 * Otherwise, idle callbacks can misfire.
129 /* Don't trace irqs off for idle */
130 stop_critical_timings();
132 start_critical_timings();
133 /* In many cases the interrupt that ended idle
134 has already called exit_idle. But some idle
135 loops can be woken up without interrupt. */
139 tick_nohz_restart_sched_tick();
140 preempt_enable_no_resched();
146 /* Prints also some state that isn't saved in the pt_regs */
147 void __show_regs(struct pt_regs *regs, int all)
149 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
150 unsigned long d0, d1, d2, d3, d6, d7;
151 unsigned int fsindex, gsindex;
152 unsigned int ds, cs, es;
156 printk(KERN_INFO "Pid: %d, comm: %.20s %s %s %.*s\n",
157 current->pid, current->comm, print_tainted(),
158 init_utsname()->release,
159 (int)strcspn(init_utsname()->version, " "),
160 init_utsname()->version);
161 printk(KERN_INFO "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
162 printk_address(regs->ip, 1);
163 printk(KERN_INFO "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss,
164 regs->sp, regs->flags);
165 printk(KERN_INFO "RAX: %016lx RBX: %016lx RCX: %016lx\n",
166 regs->ax, regs->bx, regs->cx);
167 printk(KERN_INFO "RDX: %016lx RSI: %016lx RDI: %016lx\n",
168 regs->dx, regs->si, regs->di);
169 printk(KERN_INFO "RBP: %016lx R08: %016lx R09: %016lx\n",
170 regs->bp, regs->r8, regs->r9);
171 printk(KERN_INFO "R10: %016lx R11: %016lx R12: %016lx\n",
172 regs->r10, regs->r11, regs->r12);
173 printk(KERN_INFO "R13: %016lx R14: %016lx R15: %016lx\n",
174 regs->r13, regs->r14, regs->r15);
176 asm("movl %%ds,%0" : "=r" (ds));
177 asm("movl %%cs,%0" : "=r" (cs));
178 asm("movl %%es,%0" : "=r" (es));
179 asm("movl %%fs,%0" : "=r" (fsindex));
180 asm("movl %%gs,%0" : "=r" (gsindex));
182 rdmsrl(MSR_FS_BASE, fs);
183 rdmsrl(MSR_GS_BASE, gs);
184 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
194 printk(KERN_INFO "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
195 fs, fsindex, gs, gsindex, shadowgs);
196 printk(KERN_INFO "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
198 printk(KERN_INFO "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
204 printk(KERN_INFO "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
208 printk(KERN_INFO "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
211 void show_regs(struct pt_regs *regs)
213 printk(KERN_INFO "CPU %d:", smp_processor_id());
214 __show_regs(regs, 1);
215 show_trace(NULL, regs, (void *)(regs + 1), regs->bp);
219 * Free current thread data structures etc..
221 void exit_thread(void)
223 struct task_struct *me = current;
224 struct thread_struct *t = &me->thread;
226 if (me->thread.io_bitmap_ptr) {
227 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
229 kfree(t->io_bitmap_ptr);
230 t->io_bitmap_ptr = NULL;
231 clear_thread_flag(TIF_IO_BITMAP);
233 * Careful, clear this in the TSS too:
235 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
236 t->io_bitmap_max = 0;
240 /* Free any BTS tracers that have not been properly released. */
241 if (unlikely(current->bts)) {
242 ds_release_bts(current->bts);
245 kfree(current->bts_buffer);
246 current->bts_buffer = NULL;
247 current->bts_size = 0;
249 #endif /* CONFIG_X86_DS */
252 void flush_thread(void)
254 struct task_struct *tsk = current;
256 if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) {
257 clear_tsk_thread_flag(tsk, TIF_ABI_PENDING);
258 if (test_tsk_thread_flag(tsk, TIF_IA32)) {
259 clear_tsk_thread_flag(tsk, TIF_IA32);
261 set_tsk_thread_flag(tsk, TIF_IA32);
262 current_thread_info()->status |= TS_COMPAT;
265 clear_tsk_thread_flag(tsk, TIF_DEBUG);
267 tsk->thread.debugreg0 = 0;
268 tsk->thread.debugreg1 = 0;
269 tsk->thread.debugreg2 = 0;
270 tsk->thread.debugreg3 = 0;
271 tsk->thread.debugreg6 = 0;
272 tsk->thread.debugreg7 = 0;
273 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
275 * Forget coprocessor state..
277 tsk->fpu_counter = 0;
282 void release_thread(struct task_struct *dead_task)
285 if (dead_task->mm->context.size) {
286 printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
288 dead_task->mm->context.ldt,
289 dead_task->mm->context.size);
295 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
297 struct user_desc ud = {
304 struct desc_struct *desc = t->thread.tls_array;
309 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
311 return get_desc_base(&t->thread.tls_array[tls]);
315 * This gets called before we allocate a new thread and copy
316 * the current task into it.
318 void prepare_to_copy(struct task_struct *tsk)
323 int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
324 unsigned long unused,
325 struct task_struct *p, struct pt_regs *regs)
328 struct pt_regs *childregs;
329 struct task_struct *me = current;
331 childregs = ((struct pt_regs *)
332 (THREAD_SIZE + task_stack_page(p))) - 1;
338 childregs->sp = (unsigned long)childregs;
340 p->thread.sp = (unsigned long) childregs;
341 p->thread.sp0 = (unsigned long) (childregs+1);
342 p->thread.usersp = me->thread.usersp;
344 set_tsk_thread_flag(p, TIF_FORK);
346 p->thread.fs = me->thread.fs;
347 p->thread.gs = me->thread.gs;
349 savesegment(gs, p->thread.gsindex);
350 savesegment(fs, p->thread.fsindex);
351 savesegment(es, p->thread.es);
352 savesegment(ds, p->thread.ds);
354 if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
355 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
356 if (!p->thread.io_bitmap_ptr) {
357 p->thread.io_bitmap_max = 0;
360 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
362 set_tsk_thread_flag(p, TIF_IO_BITMAP);
366 * Set a new TLS for the child thread?
368 if (clone_flags & CLONE_SETTLS) {
369 #ifdef CONFIG_IA32_EMULATION
370 if (test_thread_flag(TIF_IA32))
371 err = do_set_thread_area(p, -1,
372 (struct user_desc __user *)childregs->si, 0);
375 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
381 if (err && p->thread.io_bitmap_ptr) {
382 kfree(p->thread.io_bitmap_ptr);
383 p->thread.io_bitmap_max = 0;
389 start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
397 write_pda(oldrsp, new_sp);
398 regs->cs = __USER_CS;
399 regs->ss = __USER_DS;
403 * Free the old FP and other extended state
405 free_thread_xstate(current);
407 EXPORT_SYMBOL_GPL(start_thread);
409 static void hard_disable_TSC(void)
411 write_cr4(read_cr4() | X86_CR4_TSD);
414 void disable_TSC(void)
417 if (!test_and_set_thread_flag(TIF_NOTSC))
419 * Must flip the CPU state synchronously with
420 * TIF_NOTSC in the current running context.
426 static void hard_enable_TSC(void)
428 write_cr4(read_cr4() & ~X86_CR4_TSD);
431 static void enable_TSC(void)
434 if (test_and_clear_thread_flag(TIF_NOTSC))
436 * Must flip the CPU state synchronously with
437 * TIF_NOTSC in the current running context.
443 int get_tsc_mode(unsigned long adr)
447 if (test_thread_flag(TIF_NOTSC))
448 val = PR_TSC_SIGSEGV;
452 return put_user(val, (unsigned int __user *)adr);
455 int set_tsc_mode(unsigned int val)
457 if (val == PR_TSC_SIGSEGV)
459 else if (val == PR_TSC_ENABLE)
468 * This special macro can be used to load a debugging register
470 #define loaddebug(thread, r) set_debugreg(thread->debugreg ## r, r)
472 static inline void __switch_to_xtra(struct task_struct *prev_p,
473 struct task_struct *next_p,
474 struct tss_struct *tss)
476 struct thread_struct *prev, *next;
478 prev = &prev_p->thread,
479 next = &next_p->thread;
481 if (test_tsk_thread_flag(next_p, TIF_DS_AREA_MSR) ||
482 test_tsk_thread_flag(prev_p, TIF_DS_AREA_MSR))
483 ds_switch_to(prev_p, next_p);
484 else if (next->debugctlmsr != prev->debugctlmsr)
485 update_debugctlmsr(next->debugctlmsr);
487 if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
497 if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^
498 test_tsk_thread_flag(next_p, TIF_NOTSC)) {
499 /* prev and next are different */
500 if (test_tsk_thread_flag(next_p, TIF_NOTSC))
506 if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
508 * Copy the relevant range of the IO bitmap.
509 * Normally this is 128 bytes or less:
511 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
512 max(prev->io_bitmap_max, next->io_bitmap_max));
513 } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
515 * Clear any possible leftover bits:
517 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
522 * switch_to(x,y) should switch tasks from x to y.
524 * This could still be optimized:
525 * - fold all the options into a flag word and test it with a single test.
526 * - could test fs/gs bitsliced
528 * Kprobes not supported here. Set the probe on schedule instead.
529 * Function graph tracer not supported too.
531 __notrace_funcgraph struct task_struct *
532 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
534 struct thread_struct *prev = &prev_p->thread;
535 struct thread_struct *next = &next_p->thread;
536 int cpu = smp_processor_id();
537 struct tss_struct *tss = &per_cpu(init_tss, cpu);
538 unsigned fsindex, gsindex;
540 /* we're going to use this soon, after a few expensive things */
541 if (next_p->fpu_counter > 5)
542 prefetch(next->xstate);
545 * Reload esp0, LDT and the page table pointer:
551 * This won't pick up thread selector changes, but I guess that is ok.
553 savesegment(es, prev->es);
554 if (unlikely(next->es | prev->es))
555 loadsegment(es, next->es);
557 savesegment(ds, prev->ds);
558 if (unlikely(next->ds | prev->ds))
559 loadsegment(ds, next->ds);
562 /* We must save %fs and %gs before load_TLS() because
563 * %fs and %gs may be cleared by load_TLS().
565 * (e.g. xen_load_tls())
567 savesegment(fs, fsindex);
568 savesegment(gs, gsindex);
573 * Leave lazy mode, flushing any hypercalls made here.
574 * This must be done before restoring TLS segments so
575 * the GDT and LDT are properly updated, and must be
576 * done before math_state_restore, so the TS bit is up
579 arch_leave_lazy_cpu_mode();
584 * Segment register != 0 always requires a reload. Also
585 * reload when it has changed. When prev process used 64bit
586 * base always reload to avoid an information leak.
588 if (unlikely(fsindex | next->fsindex | prev->fs)) {
589 loadsegment(fs, next->fsindex);
591 * Check if the user used a selector != 0; if yes
592 * clear 64bit base, since overloaded base is always
593 * mapped to the Null selector
598 /* when next process has a 64bit base use it */
600 wrmsrl(MSR_FS_BASE, next->fs);
601 prev->fsindex = fsindex;
603 if (unlikely(gsindex | next->gsindex | prev->gs)) {
604 load_gs_index(next->gsindex);
609 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
610 prev->gsindex = gsindex;
612 /* Must be after DS reload */
616 * Switch the PDA and FPU contexts.
618 prev->usersp = read_pda(oldrsp);
619 write_pda(oldrsp, next->usersp);
620 write_pda(pcurrent, next_p);
622 write_pda(kernelstack,
623 (unsigned long)task_stack_page(next_p) +
624 THREAD_SIZE - PDA_STACKOFFSET);
625 #ifdef CONFIG_CC_STACKPROTECTOR
626 write_pda(stack_canary, next_p->stack_canary);
628 * Build time only check to make sure the stack_canary is at
629 * offset 40 in the pda; this is a gcc ABI requirement
631 BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
635 * Now maybe reload the debug registers and handle I/O bitmaps
637 if (unlikely(task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT ||
638 task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV))
639 __switch_to_xtra(prev_p, next_p, tss);
641 /* If the task has used fpu the last 5 timeslices, just do a full
642 * restore of the math state immediately to avoid the trap; the
643 * chances of needing FPU soon are obviously high now
645 * tsk_used_math() checks prevent calling math_state_restore(),
646 * which can sleep in the case of !tsk_used_math()
648 if (tsk_used_math(next_p) && next_p->fpu_counter > 5)
649 math_state_restore();
654 * sys_execve() executes a new program.
657 long sys_execve(char __user *name, char __user * __user *argv,
658 char __user * __user *envp, struct pt_regs *regs)
663 filename = getname(name);
664 error = PTR_ERR(filename);
665 if (IS_ERR(filename))
667 error = do_execve(filename, argv, envp, regs);
672 void set_personality_64bit(void)
674 /* inherit personality from parent */
676 /* Make sure to be in 64bit mode */
677 clear_thread_flag(TIF_IA32);
679 /* TBD: overwrites user setup. Should have two bits.
680 But 64bit processes have always behaved this way,
681 so it's not too bad. The main problem is just that
682 32bit childs are affected again. */
683 current->personality &= ~READ_IMPLIES_EXEC;
686 asmlinkage long sys_fork(struct pt_regs *regs)
688 return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL);
692 sys_clone(unsigned long clone_flags, unsigned long newsp,
693 void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
697 return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
701 * This is trivial, and on the face of it looks like it
702 * could equally well be done in user mode.
704 * Not so, for quite unobvious reasons - register pressure.
705 * In user mode vfork() cannot have a stack frame, and if
706 * done by calling the "clone()" system call directly, you
707 * do not have enough call-clobbered registers to hold all
708 * the information you need.
710 asmlinkage long sys_vfork(struct pt_regs *regs)
712 return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0,
716 unsigned long get_wchan(struct task_struct *p)
722 if (!p || p == current || p->state == TASK_RUNNING)
724 stack = (unsigned long)task_stack_page(p);
725 if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
727 fp = *(u64 *)(p->thread.sp);
729 if (fp < (unsigned long)stack ||
730 fp >= (unsigned long)stack+THREAD_SIZE)
733 if (!in_sched_functions(ip))
736 } while (count++ < 16);
740 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
743 int doit = task == current;
748 if (addr >= TASK_SIZE_OF(task))
751 /* handle small bases via the GDT because that's faster to
753 if (addr <= 0xffffffff) {
754 set_32bit_tls(task, GS_TLS, addr);
756 load_TLS(&task->thread, cpu);
757 load_gs_index(GS_TLS_SEL);
759 task->thread.gsindex = GS_TLS_SEL;
762 task->thread.gsindex = 0;
763 task->thread.gs = addr;
766 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
772 /* Not strictly needed for fs, but do it for symmetry
774 if (addr >= TASK_SIZE_OF(task))
777 /* handle small bases via the GDT because that's faster to
779 if (addr <= 0xffffffff) {
780 set_32bit_tls(task, FS_TLS, addr);
782 load_TLS(&task->thread, cpu);
783 loadsegment(fs, FS_TLS_SEL);
785 task->thread.fsindex = FS_TLS_SEL;
788 task->thread.fsindex = 0;
789 task->thread.fs = addr;
791 /* set the selector to 0 to not confuse
794 ret = checking_wrmsrl(MSR_FS_BASE, addr);
801 if (task->thread.fsindex == FS_TLS_SEL)
802 base = read_32bit_tls(task, FS_TLS);
804 rdmsrl(MSR_FS_BASE, base);
806 base = task->thread.fs;
807 ret = put_user(base, (unsigned long __user *)addr);
813 if (task->thread.gsindex == GS_TLS_SEL)
814 base = read_32bit_tls(task, GS_TLS);
816 savesegment(gs, gsindex);
818 rdmsrl(MSR_KERNEL_GS_BASE, base);
820 base = task->thread.gs;
822 base = task->thread.gs;
823 ret = put_user(base, (unsigned long __user *)addr);
835 long sys_arch_prctl(int code, unsigned long addr)
837 return do_arch_prctl(current, code, addr);
840 unsigned long arch_align_stack(unsigned long sp)
842 if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
843 sp -= get_random_int() % 8192;
847 unsigned long arch_randomize_brk(struct mm_struct *mm)
849 unsigned long range_end = mm->brk + 0x02000000;
850 return randomize_range(mm->brk, range_end, 0) ? : mm->brk;