2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/debugfs.h>
21 #include <linux/hardirq.h>
22 #include <linux/kthread.h>
23 #include <linux/uaccess.h>
24 #include <linux/kprobes.h>
25 #include <linux/ftrace.h>
26 #include <linux/sysctl.h>
27 #include <linux/ctype.h>
28 #include <linux/hash.h>
29 #include <linux/list.h>
31 #include <asm/ftrace.h>
35 /* ftrace_enabled is a method to turn ftrace on or off */
36 int ftrace_enabled __read_mostly;
37 static int last_ftrace_enabled;
40 * ftrace_disabled is set when an anomaly is discovered.
41 * ftrace_disabled is much stronger than ftrace_enabled.
43 static int ftrace_disabled __read_mostly;
45 static DEFINE_SPINLOCK(ftrace_lock);
46 static DEFINE_MUTEX(ftrace_sysctl_lock);
48 static struct ftrace_ops ftrace_list_end __read_mostly =
53 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
54 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
56 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
58 struct ftrace_ops *op = ftrace_list;
60 /* in case someone actually ports this to alpha! */
61 read_barrier_depends();
63 while (op != &ftrace_list_end) {
65 read_barrier_depends();
66 op->func(ip, parent_ip);
72 * clear_ftrace_function - reset the ftrace function
74 * This NULLs the ftrace function and in essence stops
75 * tracing. There may be lag
77 void clear_ftrace_function(void)
79 ftrace_trace_function = ftrace_stub;
82 static int __register_ftrace_function(struct ftrace_ops *ops)
84 /* should not be called from interrupt context */
85 spin_lock(&ftrace_lock);
87 ops->next = ftrace_list;
89 * We are entering ops into the ftrace_list but another
90 * CPU might be walking that list. We need to make sure
91 * the ops->next pointer is valid before another CPU sees
92 * the ops pointer included into the ftrace_list.
99 * For one func, simply call it directly.
100 * For more than one func, call the chain.
102 if (ops->next == &ftrace_list_end)
103 ftrace_trace_function = ops->func;
105 ftrace_trace_function = ftrace_list_func;
108 spin_unlock(&ftrace_lock);
113 static int __unregister_ftrace_function(struct ftrace_ops *ops)
115 struct ftrace_ops **p;
118 /* should not be called from interrupt context */
119 spin_lock(&ftrace_lock);
122 * If we are removing the last function, then simply point
123 * to the ftrace_stub.
125 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
126 ftrace_trace_function = ftrace_stub;
127 ftrace_list = &ftrace_list_end;
131 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
142 if (ftrace_enabled) {
143 /* If we only have one func left, then call that directly */
144 if (ftrace_list == &ftrace_list_end ||
145 ftrace_list->next == &ftrace_list_end)
146 ftrace_trace_function = ftrace_list->func;
150 spin_unlock(&ftrace_lock);
155 #ifdef CONFIG_DYNAMIC_FTRACE
157 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
159 * The hash lock is only needed when the recording of the mcount
160 * callers are dynamic. That is, by the caller themselves and
161 * not recorded via the compilation.
163 static DEFINE_SPINLOCK(ftrace_hash_lock);
164 #define ftrace_hash_lock(flags) spin_lock_irqsave(&ftrace_hash_lock, flags)
165 #define ftrace_hash_unlock(flags) \
166 spin_unlock_irqrestore(&ftrace_hash_lock, flags)
168 /* This is protected via the ftrace_lock with MCOUNT_RECORD. */
169 #define ftrace_hash_lock(flags) do { (void)(flags); } while (0)
170 #define ftrace_hash_unlock(flags) do { } while(0)
174 * Since MCOUNT_ADDR may point to mcount itself, we do not want
175 * to get it confused by reading a reference in the code as we
176 * are parsing on objcopy output of text. Use a variable for
179 static unsigned long mcount_addr = MCOUNT_ADDR;
181 static struct task_struct *ftraced_task;
184 FTRACE_ENABLE_CALLS = (1 << 0),
185 FTRACE_DISABLE_CALLS = (1 << 1),
186 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
187 FTRACE_ENABLE_MCOUNT = (1 << 3),
188 FTRACE_DISABLE_MCOUNT = (1 << 4),
191 static int ftrace_filtered;
192 static int tracing_on;
193 static int frozen_record_count;
195 static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
197 static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
199 static DEFINE_MUTEX(ftraced_lock);
200 static DEFINE_MUTEX(ftrace_regex_lock);
203 struct ftrace_page *next;
205 struct dyn_ftrace records[];
208 #define ENTRIES_PER_PAGE \
209 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
211 /* estimate from running different kernels */
212 #define NR_TO_INIT 10000
214 static struct ftrace_page *ftrace_pages_start;
215 static struct ftrace_page *ftrace_pages;
217 static int ftraced_trigger;
218 static int ftraced_suspend;
219 static int ftraced_stop;
221 static int ftrace_record_suspend;
223 static struct dyn_ftrace *ftrace_free_records;
226 #ifdef CONFIG_KPROBES
227 static inline void freeze_record(struct dyn_ftrace *rec)
229 if (!(rec->flags & FTRACE_FL_FROZEN)) {
230 rec->flags |= FTRACE_FL_FROZEN;
231 frozen_record_count++;
235 static inline void unfreeze_record(struct dyn_ftrace *rec)
237 if (rec->flags & FTRACE_FL_FROZEN) {
238 rec->flags &= ~FTRACE_FL_FROZEN;
239 frozen_record_count--;
243 static inline int record_frozen(struct dyn_ftrace *rec)
245 return rec->flags & FTRACE_FL_FROZEN;
248 # define freeze_record(rec) ({ 0; })
249 # define unfreeze_record(rec) ({ 0; })
250 # define record_frozen(rec) ({ 0; })
251 #endif /* CONFIG_KPROBES */
253 int skip_trace(unsigned long ip)
256 struct dyn_ftrace *rec;
257 struct hlist_node *t;
258 struct hlist_head *head;
260 if (frozen_record_count == 0)
263 head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
264 hlist_for_each_entry_rcu(rec, t, head, node) {
266 if (record_frozen(rec)) {
267 if (rec->flags & FTRACE_FL_FAILED)
270 if (!(rec->flags & FTRACE_FL_CONVERTED))
273 if (!tracing_on || !ftrace_enabled)
276 if (ftrace_filtered) {
277 fl = rec->flags & (FTRACE_FL_FILTER |
279 if (!fl || (fl & FTRACE_FL_NOTRACE))
291 ftrace_ip_in_hash(unsigned long ip, unsigned long key)
293 struct dyn_ftrace *p;
294 struct hlist_node *t;
297 hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
308 ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
310 hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
313 /* called from kstop_machine */
314 static inline void ftrace_del_hash(struct dyn_ftrace *node)
316 hlist_del(&node->node);
319 static void ftrace_free_rec(struct dyn_ftrace *rec)
321 rec->ip = (unsigned long)ftrace_free_records;
322 ftrace_free_records = rec;
323 rec->flags |= FTRACE_FL_FREE;
326 void ftrace_release(void *start, unsigned long size)
328 struct dyn_ftrace *rec;
329 struct ftrace_page *pg;
330 unsigned long s = (unsigned long)start;
331 unsigned long e = s + size;
334 if (ftrace_disabled || !start)
337 /* should not be called from interrupt context */
338 spin_lock(&ftrace_lock);
340 for (pg = ftrace_pages_start; pg; pg = pg->next) {
341 for (i = 0; i < pg->index; i++) {
342 rec = &pg->records[i];
344 if ((rec->ip >= s) && (rec->ip < e))
345 ftrace_free_rec(rec);
348 spin_unlock(&ftrace_lock);
352 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
354 struct dyn_ftrace *rec;
356 /* First check for freed records */
357 if (ftrace_free_records) {
358 rec = ftrace_free_records;
360 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
362 ftrace_free_records = NULL;
368 ftrace_free_records = (void *)rec->ip;
369 memset(rec, 0, sizeof(*rec));
373 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
374 if (!ftrace_pages->next)
376 ftrace_pages = ftrace_pages->next;
379 return &ftrace_pages->records[ftrace_pages->index++];
383 ftrace_record_ip(unsigned long ip)
385 struct dyn_ftrace *node;
391 if (!ftrace_enabled || ftrace_disabled)
394 resched = need_resched();
395 preempt_disable_notrace();
398 * We simply need to protect against recursion.
399 * Use the the raw version of smp_processor_id and not
400 * __get_cpu_var which can call debug hooks that can
401 * cause a recursive crash here.
403 cpu = raw_smp_processor_id();
404 per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
405 if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
408 if (unlikely(ftrace_record_suspend))
411 key = hash_long(ip, FTRACE_HASHBITS);
413 WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
415 if (ftrace_ip_in_hash(ip, key))
418 ftrace_hash_lock(flags);
420 /* This ip may have hit the hash before the lock */
421 if (ftrace_ip_in_hash(ip, key))
424 node = ftrace_alloc_dyn_node(ip);
430 ftrace_add_hash(node, key);
435 ftrace_hash_unlock(flags);
437 per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
439 /* prevent recursion with scheduler */
441 preempt_enable_no_resched_notrace();
443 preempt_enable_notrace();
446 #define FTRACE_ADDR ((long)(ftrace_caller))
449 __ftrace_replace_code(struct dyn_ftrace *rec,
450 unsigned char *old, unsigned char *new, int enable)
452 unsigned long ip, fl;
456 if (ftrace_filtered && enable) {
458 * If filtering is on:
460 * If this record is set to be filtered and
461 * is enabled then do nothing.
463 * If this record is set to be filtered and
464 * it is not enabled, enable it.
466 * If this record is not set to be filtered
467 * and it is not enabled do nothing.
469 * If this record is set not to trace then
472 * If this record is set not to trace and
473 * it is enabled then disable it.
475 * If this record is not set to be filtered and
476 * it is enabled, disable it.
479 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
482 if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
483 (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
484 !fl || (fl == FTRACE_FL_NOTRACE))
488 * If it is enabled disable it,
489 * otherwise enable it!
491 if (fl & FTRACE_FL_ENABLED) {
492 /* swap new and old */
494 old = ftrace_call_replace(ip, FTRACE_ADDR);
495 rec->flags &= ~FTRACE_FL_ENABLED;
497 new = ftrace_call_replace(ip, FTRACE_ADDR);
498 rec->flags |= FTRACE_FL_ENABLED;
504 * If this record is set not to trace and is
505 * not enabled, do nothing.
507 fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
508 if (fl == FTRACE_FL_NOTRACE)
511 new = ftrace_call_replace(ip, FTRACE_ADDR);
513 old = ftrace_call_replace(ip, FTRACE_ADDR);
516 if (rec->flags & FTRACE_FL_ENABLED)
518 rec->flags |= FTRACE_FL_ENABLED;
520 if (!(rec->flags & FTRACE_FL_ENABLED))
522 rec->flags &= ~FTRACE_FL_ENABLED;
526 return ftrace_modify_code(ip, old, new);
529 static void ftrace_replace_code(int enable)
532 unsigned char *new = NULL, *old = NULL;
533 struct dyn_ftrace *rec;
534 struct ftrace_page *pg;
537 old = ftrace_nop_replace();
539 new = ftrace_nop_replace();
541 for (pg = ftrace_pages_start; pg; pg = pg->next) {
542 for (i = 0; i < pg->index; i++) {
543 rec = &pg->records[i];
545 /* don't modify code that has already faulted */
546 if (rec->flags & FTRACE_FL_FAILED)
549 /* ignore updates to this record's mcount site */
550 if (get_kprobe((void *)rec->ip)) {
554 unfreeze_record(rec);
557 failed = __ftrace_replace_code(rec, old, new, enable);
558 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
559 rec->flags |= FTRACE_FL_FAILED;
560 if ((system_state == SYSTEM_BOOTING) ||
561 !core_kernel_text(rec->ip)) {
562 ftrace_del_hash(rec);
563 ftrace_free_rec(rec);
570 static void ftrace_shutdown_replenish(void)
572 if (ftrace_pages->next)
575 /* allocate another page */
576 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
580 ftrace_code_disable(struct dyn_ftrace *rec)
583 unsigned char *nop, *call;
588 nop = ftrace_nop_replace();
589 call = ftrace_call_replace(ip, mcount_addr);
591 failed = ftrace_modify_code(ip, call, nop);
593 rec->flags |= FTRACE_FL_FAILED;
599 static int __ftrace_update_code(void *ignore);
601 static int __ftrace_modify_code(void *data)
606 if (*command & FTRACE_ENABLE_CALLS) {
608 * Update any recorded ips now that we have the
611 __ftrace_update_code(NULL);
612 ftrace_replace_code(1);
614 } else if (*command & FTRACE_DISABLE_CALLS) {
615 ftrace_replace_code(0);
619 if (*command & FTRACE_UPDATE_TRACE_FUNC)
620 ftrace_update_ftrace_func(ftrace_trace_function);
622 if (*command & FTRACE_ENABLE_MCOUNT) {
623 addr = (unsigned long)ftrace_record_ip;
624 ftrace_mcount_set(&addr);
625 } else if (*command & FTRACE_DISABLE_MCOUNT) {
626 addr = (unsigned long)ftrace_stub;
627 ftrace_mcount_set(&addr);
633 static void ftrace_run_update_code(int command)
635 stop_machine(__ftrace_modify_code, &command, NULL);
638 void ftrace_disable_daemon(void)
640 /* Stop the daemon from calling kstop_machine */
641 mutex_lock(&ftraced_lock);
643 mutex_unlock(&ftraced_lock);
645 ftrace_force_update();
648 void ftrace_enable_daemon(void)
650 mutex_lock(&ftraced_lock);
652 mutex_unlock(&ftraced_lock);
654 ftrace_force_update();
657 static ftrace_func_t saved_ftrace_func;
659 static void ftrace_startup(void)
663 if (unlikely(ftrace_disabled))
666 mutex_lock(&ftraced_lock);
668 if (ftraced_suspend == 1)
669 command |= FTRACE_ENABLE_CALLS;
671 if (saved_ftrace_func != ftrace_trace_function) {
672 saved_ftrace_func = ftrace_trace_function;
673 command |= FTRACE_UPDATE_TRACE_FUNC;
676 if (!command || !ftrace_enabled)
679 ftrace_run_update_code(command);
681 mutex_unlock(&ftraced_lock);
684 static void ftrace_shutdown(void)
688 if (unlikely(ftrace_disabled))
691 mutex_lock(&ftraced_lock);
693 if (!ftraced_suspend)
694 command |= FTRACE_DISABLE_CALLS;
696 if (saved_ftrace_func != ftrace_trace_function) {
697 saved_ftrace_func = ftrace_trace_function;
698 command |= FTRACE_UPDATE_TRACE_FUNC;
701 if (!command || !ftrace_enabled)
704 ftrace_run_update_code(command);
706 mutex_unlock(&ftraced_lock);
709 static void ftrace_startup_sysctl(void)
711 int command = FTRACE_ENABLE_MCOUNT;
713 if (unlikely(ftrace_disabled))
716 mutex_lock(&ftraced_lock);
717 /* Force update next time */
718 saved_ftrace_func = NULL;
719 /* ftraced_suspend is true if we want ftrace running */
721 command |= FTRACE_ENABLE_CALLS;
723 ftrace_run_update_code(command);
724 mutex_unlock(&ftraced_lock);
727 static void ftrace_shutdown_sysctl(void)
729 int command = FTRACE_DISABLE_MCOUNT;
731 if (unlikely(ftrace_disabled))
734 mutex_lock(&ftraced_lock);
735 /* ftraced_suspend is true if ftrace is running */
737 command |= FTRACE_DISABLE_CALLS;
739 ftrace_run_update_code(command);
740 mutex_unlock(&ftraced_lock);
743 static cycle_t ftrace_update_time;
744 static unsigned long ftrace_update_cnt;
745 unsigned long ftrace_update_tot_cnt;
747 static int __ftrace_update_code(void *ignore)
749 int i, save_ftrace_enabled;
751 struct dyn_ftrace *p;
752 struct hlist_node *t, *n;
753 struct hlist_head *head, temp_list;
755 /* Don't be recording funcs now */
756 ftrace_record_suspend++;
757 save_ftrace_enabled = ftrace_enabled;
760 start = ftrace_now(raw_smp_processor_id());
761 ftrace_update_cnt = 0;
763 /* No locks needed, the machine is stopped! */
764 for (i = 0; i < FTRACE_HASHSIZE; i++) {
765 INIT_HLIST_HEAD(&temp_list);
766 head = &ftrace_hash[i];
768 /* all CPUS are stopped, we are safe to modify code */
769 hlist_for_each_entry_safe(p, t, n, head, node) {
770 /* Skip over failed records which have not been
772 if (p->flags & FTRACE_FL_FAILED)
775 /* Unconverted records are always at the head of the
776 * hash bucket. Once we encounter a converted record,
777 * simply skip over to the next bucket. Saves ftraced
778 * some processor cycles (ftrace does its bid for
779 * global warming :-p ). */
780 if (p->flags & (FTRACE_FL_CONVERTED))
783 /* Ignore updates to this record's mcount site.
784 * Reintroduce this record at the head of this
785 * bucket to attempt to "convert" it again if
786 * the kprobe on it is unregistered before the
788 if (get_kprobe((void *)p->ip)) {
790 INIT_HLIST_NODE(&p->node);
791 hlist_add_head(&p->node, &temp_list);
798 /* convert record (i.e, patch mcount-call with NOP) */
799 if (ftrace_code_disable(p)) {
800 p->flags |= FTRACE_FL_CONVERTED;
803 if ((system_state == SYSTEM_BOOTING) ||
804 !core_kernel_text(p->ip)) {
811 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
813 INIT_HLIST_NODE(&p->node);
814 hlist_add_head(&p->node, head);
818 stop = ftrace_now(raw_smp_processor_id());
819 ftrace_update_time = stop - start;
820 ftrace_update_tot_cnt += ftrace_update_cnt;
823 ftrace_enabled = save_ftrace_enabled;
824 ftrace_record_suspend--;
829 static int ftrace_update_code(void)
831 if (unlikely(ftrace_disabled) ||
832 !ftrace_enabled || !ftraced_trigger)
835 stop_machine(__ftrace_update_code, NULL, NULL);
840 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
842 struct ftrace_page *pg;
846 /* allocate a few pages */
847 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
848 if (!ftrace_pages_start)
852 * Allocate a few more pages.
854 * TODO: have some parser search vmlinux before
855 * final linking to find all calls to ftrace.
857 * a) know how many pages to allocate.
859 * b) set up the table then.
861 * The dynamic code is still necessary for
865 pg = ftrace_pages = ftrace_pages_start;
867 cnt = num_to_init / ENTRIES_PER_PAGE;
868 pr_info("ftrace: allocating %ld hash entries in %d pages\n",
871 for (i = 0; i < cnt; i++) {
872 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
874 /* If we fail, we'll try later anyway */
885 FTRACE_ITER_FILTER = (1 << 0),
886 FTRACE_ITER_CONT = (1 << 1),
887 FTRACE_ITER_NOTRACE = (1 << 2),
888 FTRACE_ITER_FAILURES = (1 << 3),
891 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
893 struct ftrace_iterator {
895 struct ftrace_page *pg;
898 unsigned char buffer[FTRACE_BUFF_MAX+1];
904 t_next(struct seq_file *m, void *v, loff_t *pos)
906 struct ftrace_iterator *iter = m->private;
907 struct dyn_ftrace *rec = NULL;
911 /* should not be called from interrupt context */
912 spin_lock(&ftrace_lock);
914 if (iter->idx >= iter->pg->index) {
915 if (iter->pg->next) {
916 iter->pg = iter->pg->next;
921 rec = &iter->pg->records[iter->idx++];
922 if ((rec->flags & FTRACE_FL_FREE) ||
924 (!(iter->flags & FTRACE_ITER_FAILURES) &&
925 (rec->flags & FTRACE_FL_FAILED)) ||
927 ((iter->flags & FTRACE_ITER_FAILURES) &&
928 !(rec->flags & FTRACE_FL_FAILED)) ||
930 ((iter->flags & FTRACE_ITER_NOTRACE) &&
931 !(rec->flags & FTRACE_FL_NOTRACE))) {
936 spin_unlock(&ftrace_lock);
943 static void *t_start(struct seq_file *m, loff_t *pos)
945 struct ftrace_iterator *iter = m->private;
949 if (*pos != iter->pos) {
950 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
954 p = t_next(m, p, &l);
960 static void t_stop(struct seq_file *m, void *p)
964 static int t_show(struct seq_file *m, void *v)
966 struct dyn_ftrace *rec = v;
967 char str[KSYM_SYMBOL_LEN];
972 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
974 seq_printf(m, "%s\n", str);
979 static struct seq_operations show_ftrace_seq_ops = {
987 ftrace_avail_open(struct inode *inode, struct file *file)
989 struct ftrace_iterator *iter;
992 if (unlikely(ftrace_disabled))
995 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
999 iter->pg = ftrace_pages_start;
1002 ret = seq_open(file, &show_ftrace_seq_ops);
1004 struct seq_file *m = file->private_data;
1014 int ftrace_avail_release(struct inode *inode, struct file *file)
1016 struct seq_file *m = (struct seq_file *)file->private_data;
1017 struct ftrace_iterator *iter = m->private;
1019 seq_release(inode, file);
1026 ftrace_failures_open(struct inode *inode, struct file *file)
1030 struct ftrace_iterator *iter;
1032 ret = ftrace_avail_open(inode, file);
1034 m = (struct seq_file *)file->private_data;
1035 iter = (struct ftrace_iterator *)m->private;
1036 iter->flags = FTRACE_ITER_FAILURES;
1043 static void ftrace_filter_reset(int enable)
1045 struct ftrace_page *pg;
1046 struct dyn_ftrace *rec;
1047 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1050 /* should not be called from interrupt context */
1051 spin_lock(&ftrace_lock);
1053 ftrace_filtered = 0;
1054 pg = ftrace_pages_start;
1056 for (i = 0; i < pg->index; i++) {
1057 rec = &pg->records[i];
1058 if (rec->flags & FTRACE_FL_FAILED)
1060 rec->flags &= ~type;
1064 spin_unlock(&ftrace_lock);
1068 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1070 struct ftrace_iterator *iter;
1073 if (unlikely(ftrace_disabled))
1076 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1080 mutex_lock(&ftrace_regex_lock);
1081 if ((file->f_mode & FMODE_WRITE) &&
1082 !(file->f_flags & O_APPEND))
1083 ftrace_filter_reset(enable);
1085 if (file->f_mode & FMODE_READ) {
1086 iter->pg = ftrace_pages_start;
1088 iter->flags = enable ? FTRACE_ITER_FILTER :
1089 FTRACE_ITER_NOTRACE;
1091 ret = seq_open(file, &show_ftrace_seq_ops);
1093 struct seq_file *m = file->private_data;
1098 file->private_data = iter;
1099 mutex_unlock(&ftrace_regex_lock);
1105 ftrace_filter_open(struct inode *inode, struct file *file)
1107 return ftrace_regex_open(inode, file, 1);
1111 ftrace_notrace_open(struct inode *inode, struct file *file)
1113 return ftrace_regex_open(inode, file, 0);
1117 ftrace_regex_read(struct file *file, char __user *ubuf,
1118 size_t cnt, loff_t *ppos)
1120 if (file->f_mode & FMODE_READ)
1121 return seq_read(file, ubuf, cnt, ppos);
1127 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1131 if (file->f_mode & FMODE_READ)
1132 ret = seq_lseek(file, offset, origin);
1134 file->f_pos = ret = 1;
1147 ftrace_match(unsigned char *buff, int len, int enable)
1149 char str[KSYM_SYMBOL_LEN];
1150 char *search = NULL;
1151 struct ftrace_page *pg;
1152 struct dyn_ftrace *rec;
1153 int type = MATCH_FULL;
1154 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1155 unsigned i, match = 0, search_len = 0;
1157 for (i = 0; i < len; i++) {
1158 if (buff[i] == '*') {
1160 search = buff + i + 1;
1161 type = MATCH_END_ONLY;
1162 search_len = len - (i + 1);
1164 if (type == MATCH_END_ONLY) {
1165 type = MATCH_MIDDLE_ONLY;
1168 type = MATCH_FRONT_ONLY;
1176 /* should not be called from interrupt context */
1177 spin_lock(&ftrace_lock);
1179 ftrace_filtered = 1;
1180 pg = ftrace_pages_start;
1182 for (i = 0; i < pg->index; i++) {
1186 rec = &pg->records[i];
1187 if (rec->flags & FTRACE_FL_FAILED)
1189 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1192 if (strcmp(str, buff) == 0)
1195 case MATCH_FRONT_ONLY:
1196 if (memcmp(str, buff, match) == 0)
1199 case MATCH_MIDDLE_ONLY:
1200 if (strstr(str, search))
1203 case MATCH_END_ONLY:
1204 ptr = strstr(str, search);
1205 if (ptr && (ptr[search_len] == 0))
1214 spin_unlock(&ftrace_lock);
1218 ftrace_regex_write(struct file *file, const char __user *ubuf,
1219 size_t cnt, loff_t *ppos, int enable)
1221 struct ftrace_iterator *iter;
1226 if (!cnt || cnt < 0)
1229 mutex_lock(&ftrace_regex_lock);
1231 if (file->f_mode & FMODE_READ) {
1232 struct seq_file *m = file->private_data;
1235 iter = file->private_data;
1238 iter->flags &= ~FTRACE_ITER_CONT;
1239 iter->buffer_idx = 0;
1242 ret = get_user(ch, ubuf++);
1248 if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1249 /* skip white space */
1250 while (cnt && isspace(ch)) {
1251 ret = get_user(ch, ubuf++);
1259 file->f_pos += read;
1264 iter->buffer_idx = 0;
1267 while (cnt && !isspace(ch)) {
1268 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1269 iter->buffer[iter->buffer_idx++] = ch;
1274 ret = get_user(ch, ubuf++);
1283 iter->buffer[iter->buffer_idx] = 0;
1284 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1285 iter->buffer_idx = 0;
1287 iter->flags |= FTRACE_ITER_CONT;
1290 file->f_pos += read;
1294 mutex_unlock(&ftrace_regex_lock);
1300 ftrace_filter_write(struct file *file, const char __user *ubuf,
1301 size_t cnt, loff_t *ppos)
1303 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1307 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1308 size_t cnt, loff_t *ppos)
1310 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1314 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1316 if (unlikely(ftrace_disabled))
1319 mutex_lock(&ftrace_regex_lock);
1321 ftrace_filter_reset(enable);
1323 ftrace_match(buf, len, enable);
1324 mutex_unlock(&ftrace_regex_lock);
1328 * ftrace_set_filter - set a function to filter on in ftrace
1329 * @buf - the string that holds the function filter text.
1330 * @len - the length of the string.
1331 * @reset - non zero to reset all filters before applying this filter.
1333 * Filters denote which functions should be enabled when tracing is enabled.
1334 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1336 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1338 ftrace_set_regex(buf, len, reset, 1);
1342 * ftrace_set_notrace - set a function to not trace in ftrace
1343 * @buf - the string that holds the function notrace text.
1344 * @len - the length of the string.
1345 * @reset - non zero to reset all filters before applying this filter.
1347 * Notrace Filters denote which functions should not be enabled when tracing
1348 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1351 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1353 ftrace_set_regex(buf, len, reset, 0);
1357 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1359 struct seq_file *m = (struct seq_file *)file->private_data;
1360 struct ftrace_iterator *iter;
1362 mutex_lock(&ftrace_regex_lock);
1363 if (file->f_mode & FMODE_READ) {
1366 seq_release(inode, file);
1368 iter = file->private_data;
1370 if (iter->buffer_idx) {
1372 iter->buffer[iter->buffer_idx] = 0;
1373 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1376 mutex_lock(&ftrace_sysctl_lock);
1377 mutex_lock(&ftraced_lock);
1378 if (iter->filtered && ftraced_suspend && ftrace_enabled)
1379 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1380 mutex_unlock(&ftraced_lock);
1381 mutex_unlock(&ftrace_sysctl_lock);
1384 mutex_unlock(&ftrace_regex_lock);
1389 ftrace_filter_release(struct inode *inode, struct file *file)
1391 return ftrace_regex_release(inode, file, 1);
1395 ftrace_notrace_release(struct inode *inode, struct file *file)
1397 return ftrace_regex_release(inode, file, 0);
1401 ftraced_read(struct file *filp, char __user *ubuf,
1402 size_t cnt, loff_t *ppos)
1404 /* don't worry about races */
1405 char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1406 int r = strlen(buf);
1408 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1412 ftraced_write(struct file *filp, const char __user *ubuf,
1413 size_t cnt, loff_t *ppos)
1419 if (cnt >= sizeof(buf))
1422 if (copy_from_user(&buf, ubuf, cnt))
1425 if (strncmp(buf, "enable", 6) == 0)
1427 else if (strncmp(buf, "disable", 7) == 0)
1432 ret = strict_strtoul(buf, 10, &val);
1440 ftrace_enable_daemon();
1442 ftrace_disable_daemon();
1449 static struct file_operations ftrace_avail_fops = {
1450 .open = ftrace_avail_open,
1452 .llseek = seq_lseek,
1453 .release = ftrace_avail_release,
1456 static struct file_operations ftrace_failures_fops = {
1457 .open = ftrace_failures_open,
1459 .llseek = seq_lseek,
1460 .release = ftrace_avail_release,
1463 static struct file_operations ftrace_filter_fops = {
1464 .open = ftrace_filter_open,
1465 .read = ftrace_regex_read,
1466 .write = ftrace_filter_write,
1467 .llseek = ftrace_regex_lseek,
1468 .release = ftrace_filter_release,
1471 static struct file_operations ftrace_notrace_fops = {
1472 .open = ftrace_notrace_open,
1473 .read = ftrace_regex_read,
1474 .write = ftrace_notrace_write,
1475 .llseek = ftrace_regex_lseek,
1476 .release = ftrace_notrace_release,
1479 static struct file_operations ftraced_fops = {
1480 .open = tracing_open_generic,
1481 .read = ftraced_read,
1482 .write = ftraced_write,
1486 * ftrace_force_update - force an update to all recording ftrace functions
1488 int ftrace_force_update(void)
1492 if (unlikely(ftrace_disabled))
1495 mutex_lock(&ftrace_sysctl_lock);
1496 mutex_lock(&ftraced_lock);
1499 * If ftraced_trigger is not set, then there is nothing
1502 if (ftraced_trigger && !ftrace_update_code())
1505 mutex_unlock(&ftraced_lock);
1506 mutex_unlock(&ftrace_sysctl_lock);
1511 static void ftrace_force_shutdown(void)
1513 struct task_struct *task;
1514 int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1516 mutex_lock(&ftraced_lock);
1517 task = ftraced_task;
1518 ftraced_task = NULL;
1519 ftraced_suspend = -1;
1520 ftrace_run_update_code(command);
1521 mutex_unlock(&ftraced_lock);
1527 static __init int ftrace_init_debugfs(void)
1529 struct dentry *d_tracer;
1530 struct dentry *entry;
1532 d_tracer = tracing_init_dentry();
1534 entry = debugfs_create_file("available_filter_functions", 0444,
1535 d_tracer, NULL, &ftrace_avail_fops);
1537 pr_warning("Could not create debugfs "
1538 "'available_filter_functions' entry\n");
1540 entry = debugfs_create_file("failures", 0444,
1541 d_tracer, NULL, &ftrace_failures_fops);
1543 pr_warning("Could not create debugfs 'failures' entry\n");
1545 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1546 NULL, &ftrace_filter_fops);
1548 pr_warning("Could not create debugfs "
1549 "'set_ftrace_filter' entry\n");
1551 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1552 NULL, &ftrace_notrace_fops);
1554 pr_warning("Could not create debugfs "
1555 "'set_ftrace_notrace' entry\n");
1557 entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1558 NULL, &ftraced_fops);
1560 pr_warning("Could not create debugfs "
1561 "'ftraced_enabled' entry\n");
1565 fs_initcall(ftrace_init_debugfs);
1567 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
1568 static int ftrace_convert_nops(unsigned long *start,
1573 unsigned long flags;
1577 addr = ftrace_call_adjust(*p++);
1578 /* should not be called from interrupt context */
1579 spin_lock(&ftrace_lock);
1580 ftrace_record_ip(addr);
1581 spin_unlock(&ftrace_lock);
1582 ftrace_shutdown_replenish();
1586 local_irq_save(flags);
1587 __ftrace_update_code(p);
1588 local_irq_restore(flags);
1593 void ftrace_init_module(unsigned long *start, unsigned long *end)
1595 if (ftrace_disabled || start == end)
1597 ftrace_convert_nops(start, end);
1600 extern unsigned long __start_mcount_loc[];
1601 extern unsigned long __stop_mcount_loc[];
1603 void __init ftrace_init(void)
1605 unsigned long count, addr, flags;
1608 /* Keep the ftrace pointer to the stub */
1609 addr = (unsigned long)ftrace_stub;
1611 local_irq_save(flags);
1612 ftrace_dyn_arch_init(&addr);
1613 local_irq_restore(flags);
1615 /* ftrace_dyn_arch_init places the return code in addr */
1619 count = __stop_mcount_loc - __start_mcount_loc;
1621 ret = ftrace_dyn_table_alloc(count);
1625 last_ftrace_enabled = ftrace_enabled = 1;
1627 ret = ftrace_convert_nops(__start_mcount_loc,
1632 ftrace_disabled = 1;
1634 #else /* CONFIG_FTRACE_MCOUNT_RECORD */
1635 static int ftraced(void *ignore)
1637 unsigned long usecs;
1639 while (!kthread_should_stop()) {
1641 set_current_state(TASK_INTERRUPTIBLE);
1643 /* check once a second */
1644 schedule_timeout(HZ);
1646 if (unlikely(ftrace_disabled))
1649 mutex_lock(&ftrace_sysctl_lock);
1650 mutex_lock(&ftraced_lock);
1651 if (!ftraced_suspend && !ftraced_stop &&
1652 ftrace_update_code()) {
1653 usecs = nsecs_to_usecs(ftrace_update_time);
1654 if (ftrace_update_tot_cnt > 100000) {
1655 ftrace_update_tot_cnt = 0;
1656 pr_info("hm, dftrace overflow: %lu change%s"
1657 " (%lu total) in %lu usec%s\n",
1659 ftrace_update_cnt != 1 ? "s" : "",
1660 ftrace_update_tot_cnt,
1661 usecs, usecs != 1 ? "s" : "");
1662 ftrace_disabled = 1;
1666 mutex_unlock(&ftraced_lock);
1667 mutex_unlock(&ftrace_sysctl_lock);
1669 ftrace_shutdown_replenish();
1671 __set_current_state(TASK_RUNNING);
1675 static int __init ftrace_dynamic_init(void)
1677 struct task_struct *p;
1681 addr = (unsigned long)ftrace_record_ip;
1683 stop_machine(ftrace_dyn_arch_init, &addr, NULL);
1685 /* ftrace_dyn_arch_init places the return code in addr */
1691 ret = ftrace_dyn_table_alloc(NR_TO_INIT);
1695 p = kthread_run(ftraced, NULL, "ftraced");
1701 last_ftrace_enabled = ftrace_enabled = 1;
1707 ftrace_disabled = 1;
1711 core_initcall(ftrace_dynamic_init);
1712 #endif /* CONFIG_FTRACE_MCOUNT_RECORD */
1715 # define ftrace_startup() do { } while (0)
1716 # define ftrace_shutdown() do { } while (0)
1717 # define ftrace_startup_sysctl() do { } while (0)
1718 # define ftrace_shutdown_sysctl() do { } while (0)
1719 # define ftrace_force_shutdown() do { } while (0)
1720 #endif /* CONFIG_DYNAMIC_FTRACE */
1723 * ftrace_kill_atomic - kill ftrace from critical sections
1725 * This function should be used by panic code. It stops ftrace
1726 * but in a not so nice way. If you need to simply kill ftrace
1727 * from a non-atomic section, use ftrace_kill.
1729 void ftrace_kill_atomic(void)
1731 ftrace_disabled = 1;
1733 #ifdef CONFIG_DYNAMIC_FTRACE
1734 ftraced_suspend = -1;
1736 clear_ftrace_function();
1740 * ftrace_kill - totally shutdown ftrace
1742 * This is a safety measure. If something was detected that seems
1743 * wrong, calling this function will keep ftrace from doing
1744 * any more modifications, and updates.
1745 * used when something went wrong.
1747 void ftrace_kill(void)
1749 mutex_lock(&ftrace_sysctl_lock);
1750 ftrace_disabled = 1;
1753 clear_ftrace_function();
1754 mutex_unlock(&ftrace_sysctl_lock);
1756 /* Try to totally disable ftrace */
1757 ftrace_force_shutdown();
1761 * register_ftrace_function - register a function for profiling
1762 * @ops - ops structure that holds the function for profiling.
1764 * Register a function to be called by all functions in the
1767 * Note: @ops->func and all the functions it calls must be labeled
1768 * with "notrace", otherwise it will go into a
1771 int register_ftrace_function(struct ftrace_ops *ops)
1775 if (unlikely(ftrace_disabled))
1778 mutex_lock(&ftrace_sysctl_lock);
1779 ret = __register_ftrace_function(ops);
1781 mutex_unlock(&ftrace_sysctl_lock);
1787 * unregister_ftrace_function - unresgister a function for profiling.
1788 * @ops - ops structure that holds the function to unregister
1790 * Unregister a function that was added to be called by ftrace profiling.
1792 int unregister_ftrace_function(struct ftrace_ops *ops)
1796 mutex_lock(&ftrace_sysctl_lock);
1797 ret = __unregister_ftrace_function(ops);
1799 mutex_unlock(&ftrace_sysctl_lock);
1805 ftrace_enable_sysctl(struct ctl_table *table, int write,
1806 struct file *file, void __user *buffer, size_t *lenp,
1811 if (unlikely(ftrace_disabled))
1814 mutex_lock(&ftrace_sysctl_lock);
1816 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1818 if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1821 last_ftrace_enabled = ftrace_enabled;
1823 if (ftrace_enabled) {
1825 ftrace_startup_sysctl();
1827 /* we are starting ftrace again */
1828 if (ftrace_list != &ftrace_list_end) {
1829 if (ftrace_list->next == &ftrace_list_end)
1830 ftrace_trace_function = ftrace_list->func;
1832 ftrace_trace_function = ftrace_list_func;
1836 /* stopping ftrace calls (just send to ftrace_stub) */
1837 ftrace_trace_function = ftrace_stub;
1839 ftrace_shutdown_sysctl();
1843 mutex_unlock(&ftrace_sysctl_lock);