arch/x86/kernel/tsc_32.c

   1 #include <linux/sched.h>
   2 #include <linux/clocksource.h>
   3 #include <linux/workqueue.h>
   4 #include <linux/delay.h>
   5 #include <linux/cpufreq.h>
   6 #include <linux/jiffies.h>
   7 #include <linux/init.h>
   8 #include <linux/dmi.h>
   9 #include <linux/percpu.h>
  10
  11 #include <asm/delay.h>
  12 #include <asm/tsc.h>
  13 #include <asm/io.h>
  14 #include <asm/timer.h>
  15
  16 #include "mach_timer.h"
  17
  18 extern int tsc_unstable;
  19 extern int tsc_disabled;
  20
  21 /* Accelerators for sched_clock()
  22  * convert from cycles(64bits) => nanoseconds (64bits)
  23  *  basic equation:
  24  *              ns = cycles / (freq / ns_per_sec)
  25  *              ns = cycles * (ns_per_sec / freq)
  26  *              ns = cycles * (10^9 / (cpu_khz * 10^3))
  27  *              ns = cycles * (10^6 / cpu_khz)
  28  *
  29  *      Then we use scaling math (suggested by george@mvista.com) to get:
  30  *              ns = cycles * (10^6 * SC / cpu_khz) / SC
  31  *              ns = cycles * cyc2ns_scale / SC
  32  *
  33  *      And since SC is a constant power of two, we can convert the div
  34  *  into a shift.
  35  *
  36  *  We can use khz divisor instead of mhz to keep a better precision, since
  37  *  cyc2ns_scale is limited to 10^6 * 2^10, which fits in 32 bits.
  38  *  (mathieu.desnoyers@polymtl.ca)
  39  *
  40  *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
  41  */
  42
  43 DEFINE_PER_CPU(unsigned long, cyc2ns);
  44
  45 void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
  46 {
  47         unsigned long long tsc_now, ns_now;
  48         unsigned long flags, *scale;
  49
  50         local_irq_save(flags);
  51         sched_clock_idle_sleep_event();
  52
  53         scale = &per_cpu(cyc2ns, cpu);
  54
  55         rdtscll(tsc_now);
  56         ns_now = __cycles_2_ns(tsc_now);
  57
  58         if (cpu_khz)
  59                 *scale = (NSEC_PER_MSEC << CYC2NS_SCALE_FACTOR)/cpu_khz;
  60
  61         /*
  62          * Start smoothly with the new frequency:
  63          */
  64         sched_clock_idle_wakeup_event(0);
  65         local_irq_restore(flags);
  66 }
  67
  68 #ifdef CONFIG_CPU_FREQ
  69
  70 /*
  71  * if the CPU frequency is scaled, TSC-based delays will need a different
  72  * loops_per_jiffy value to function properly.
  73  */
  74 static unsigned int ref_freq;
  75 static unsigned long loops_per_jiffy_ref;
  76 static unsigned long cpu_khz_ref;
  77
  78 static int
  79 time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data)
  80 {
  81         struct cpufreq_freqs *freq = data;
  82
  83         if (!ref_freq) {
  84                 if (!freq->old){
  85                         ref_freq = freq->new;
  86                         return 0;
  87                 }
  88                 ref_freq = freq->old;
  89                 loops_per_jiffy_ref = cpu_data(freq->cpu).loops_per_jiffy;
  90                 cpu_khz_ref = cpu_khz;
  91         }
  92
  93         if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
  94             (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) ||
  95             (val == CPUFREQ_RESUMECHANGE)) {
  96                 if (!(freq->flags & CPUFREQ_CONST_LOOPS))
  97                         cpu_data(freq->cpu).loops_per_jiffy =
  98                                 cpufreq_scale(loops_per_jiffy_ref,
  99                                                 ref_freq, freq->new);
 100
 101                 if (cpu_khz) {
 102
 103                         if (num_online_cpus() == 1)
 104                                 cpu_khz = cpufreq_scale(cpu_khz_ref,
 105                                                 ref_freq, freq->new);
 106                         if (!(freq->flags & CPUFREQ_CONST_LOOPS)) {
 107                                 tsc_khz = cpu_khz;
 108                                 set_cyc2ns_scale(cpu_khz, freq->cpu);
 109                                 /*
 110                                  * TSC based sched_clock turns
 111                                  * to junk w/ cpufreq
 112                                  */
 113                                 mark_tsc_unstable("cpufreq changes");
 114                         }
 115                 }
 116         }
 117
 118         return 0;
 119 }
 120
 121 static struct notifier_block time_cpufreq_notifier_block = {
 122         .notifier_call  = time_cpufreq_notifier
 123 };
 124
 125 static int __init cpufreq_tsc(void)
 126 {
 127         return cpufreq_register_notifier(&time_cpufreq_notifier_block,
 128                                          CPUFREQ_TRANSITION_NOTIFIER);
 129 }
 130 core_initcall(cpufreq_tsc);
 131
 132 #endif
 133
 134 /* clock source code */
 135
 136 static struct clocksource clocksource_tsc;
 137
 138 /*
 139  * We compare the TSC to the cycle_last value in the clocksource
 140  * structure to avoid a nasty time-warp issue. This can be observed in
 141  * a very small window right after one CPU updated cycle_last under
 142  * xtime lock and the other CPU reads a TSC value which is smaller
 143  * than the cycle_last reference value due to a TSC which is slighty
 144  * behind. This delta is nowhere else observable, but in that case it
 145  * results in a forward time jump in the range of hours due to the
 146  * unsigned delta calculation of the time keeping core code, which is
 147  * necessary to support wrapping clocksources like pm timer.
 148  */
 149 static cycle_t read_tsc(void)
 150 {
 151         cycle_t ret;
 152
 153         rdtscll(ret);
 154
 155         return ret >= clocksource_tsc.cycle_last ?
 156                 ret : clocksource_tsc.cycle_last;
 157 }
 158
 159 static struct clocksource clocksource_tsc = {
 160         .name                   = "tsc",
 161         .rating                 = 300,
 162         .read                   = read_tsc,
 163         .mask                   = CLOCKSOURCE_MASK(64),
 164         .mult                   = 0, /* to be set */
 165         .shift                  = 22,
 166         .flags                  = CLOCK_SOURCE_IS_CONTINUOUS |
 167                                   CLOCK_SOURCE_MUST_VERIFY,
 168 };
 169
 170 void mark_tsc_unstable(char *reason)
 171 {
 172         if (!tsc_unstable) {
 173                 tsc_unstable = 1;
 174                 printk("Marking TSC unstable due to: %s.\n", reason);
 175                 /* Can be called before registration */
 176                 if (clocksource_tsc.mult)
 177                         clocksource_change_rating(&clocksource_tsc, 0);
 178                 else
 179                         clocksource_tsc.rating = 0;
 180         }
 181 }
 182 EXPORT_SYMBOL_GPL(mark_tsc_unstable);
 183
 184 static int __init dmi_mark_tsc_unstable(const struct dmi_system_id *d)
 185 {
 186         printk(KERN_NOTICE "%s detected: marking TSC unstable.\n",
 187                d->ident);
 188         tsc_unstable = 1;
 189         return 0;
 190 }
 191
 192 /* List of systems that have known TSC problems */
 193 static struct dmi_system_id __initdata bad_tsc_dmi_table[] = {
 194         {
 195          .callback = dmi_mark_tsc_unstable,
 196          .ident = "IBM Thinkpad 380XD",
 197          .matches = {
 198                      DMI_MATCH(DMI_BOARD_VENDOR, "IBM"),
 199                      DMI_MATCH(DMI_BOARD_NAME, "2635FA0"),
 200                      },
 201          },
 202          {}
 203 };
 204
 205 /*
 206  * Make an educated guess if the TSC is trustworthy and synchronized
 207  * over all CPUs.
 208  */
 209 __cpuinit int unsynchronized_tsc(void)
 210 {
 211         if (!cpu_has_tsc || tsc_unstable)
 212                 return 1;
 213
 214         /* Anything with constant TSC should be synchronized */
 215         if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
 216                 return 0;
 217
 218         /*
 219          * Intel systems are normally all synchronized.
 220          * Exceptions must mark TSC as unstable:
 221          */
 222         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL) {
 223                 /* assume multi socket systems are not synchronized: */
 224                 if (num_possible_cpus() > 1)
 225                         tsc_unstable = 1;
 226         }
 227         return tsc_unstable;
 228 }
 229
 230 /*
 231  * Geode_LX - the OLPC CPU has a possibly a very reliable TSC
 232  */
 233 #ifdef CONFIG_MGEODE_LX
 234 /* RTSC counts during suspend */
 235 #define RTSC_SUSP 0x100
 236
 237 static void __init check_geode_tsc_reliable(void)
 238 {
 239         unsigned long res_low, res_high;
 240
 241         rdmsr_safe(MSR_GEODE_BUSCONT_CONF0, &res_low, &res_high);
 242         if (res_low & RTSC_SUSP)
 243                 clocksource_tsc.flags &= ~CLOCK_SOURCE_MUST_VERIFY;
 244 }
 245 #else
 246 static inline void check_geode_tsc_reliable(void) { }
 247 #endif
 248
 249
 250 void __init tsc_init(void)
 251 {
 252         int cpu;
 253         u64 lpj;
 254
 255         if (!cpu_has_tsc || tsc_disabled > 0)
 256                 return;
 257
 258         cpu_khz = calculate_cpu_khz();
 259         tsc_khz = cpu_khz;
 260
 261         if (!cpu_khz) {
 262                 mark_tsc_unstable("could not calculate TSC khz");
 263                 return;
 264         }
 265
 266         lpj = ((u64)tsc_khz * 1000);
 267         do_div(lpj, HZ);
 268         lpj_fine = lpj;
 269
 270         /* now allow native_sched_clock() to use rdtsc */
 271         tsc_disabled = 0;
 272
 273         printk("Detected %lu.%03lu MHz processor.\n",
 274                                 (unsigned long)cpu_khz / 1000,
 275                                 (unsigned long)cpu_khz % 1000);
 276
 277         /*
 278          * Secondary CPUs do not run through tsc_init(), so set up
 279          * all the scale factors for all CPUs, assuming the same
 280          * speed as the bootup CPU. (cpufreq notifiers will fix this
 281          * up if their speed diverges)
 282          */
 283         for_each_possible_cpu(cpu)
 284                 set_cyc2ns_scale(cpu_khz, cpu);
 285
 286         use_tsc_delay();
 287
 288         /* Check and install the TSC clocksource */
 289         dmi_check_system(bad_tsc_dmi_table);
 290
 291         unsynchronized_tsc();
 292         check_geode_tsc_reliable();
 293         clocksource_tsc.mult = clocksource_khz2mult(tsc_khz,
 294                                                     clocksource_tsc.shift);
 295         /* lower the rating if we already know its unstable: */
 296         if (check_tsc_unstable()) {
 297                 clocksource_tsc.rating = 0;
 298                 clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS;
 299         }
 300         clocksource_register(&clocksource_tsc);
 301 }