arch/x86/xen/spinlock.c

   1 /*
   2  * Split spinlock implementation out into its own file, so it can be
   3  * compiled in a FTRACE-compatible way.
   4  */
   5 #include <linux/kernel_stat.h>
   6 #include <linux/spinlock.h>
   7 #include <linux/debugfs.h>
   8 #include <linux/log2.h>
   9
  10 #include <asm/paravirt.h>
  11
  12 #include <xen/interface/xen.h>
  13 #include <xen/events.h>
  14
  15 #include "xen-ops.h"
  16 #include "debugfs.h"
  17
  18 #ifdef CONFIG_XEN_DEBUG_FS
  19 static struct xen_spinlock_stats
  20 {
  21         u64 taken;
  22         u32 taken_slow;
  23         u32 taken_slow_nested;
  24         u32 taken_slow_pickup;
  25         u32 taken_slow_spurious;
  26
  27         u64 released;
  28         u32 released_slow;
  29         u32 released_slow_kicked;
  30
  31 #define HISTO_BUCKETS   20
  32         u32 histo_spin_fast[HISTO_BUCKETS+1];
  33         u32 histo_spin[HISTO_BUCKETS+1];
  34
  35         u64 spinning_time;
  36         u64 total_time;
  37 } spinlock_stats;
  38
  39 static u8 zero_stats;
  40
  41 static unsigned lock_timeout = 1 << 10;
  42 #define TIMEOUT lock_timeout
  43
  44 static inline void check_zero(void)
  45 {
  46         if (unlikely(zero_stats)) {
  47                 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
  48                 zero_stats = 0;
  49         }
  50 }
  51
  52 #define ADD_STATS(elem, val)                    \
  53         do { check_zero(); spinlock_stats.elem += (val); } while(0)
  54
  55 static inline u64 spin_time_start(void)
  56 {
  57         return xen_clocksource_read();
  58 }
  59
  60 static void __spin_time_accum(u64 delta, u32 *array)
  61 {
  62         unsigned index = ilog2(delta);
  63
  64         check_zero();
  65
  66         if (index < HISTO_BUCKETS)
  67                 array[index]++;
  68         else
  69                 array[HISTO_BUCKETS]++;
  70 }
  71
  72 static inline void spin_time_accum_fast(u64 start)
  73 {
  74         u32 delta = xen_clocksource_read() - start;
  75
  76         __spin_time_accum(delta, spinlock_stats.histo_spin_fast);
  77         spinlock_stats.spinning_time += delta;
  78 }
  79
  80 static inline void spin_time_accum(u64 start)
  81 {
  82         u32 delta = xen_clocksource_read() - start;
  83
  84         __spin_time_accum(delta, spinlock_stats.histo_spin);
  85         spinlock_stats.total_time += delta;
  86 }
  87 #else  /* !CONFIG_XEN_DEBUG_FS */
  88 #define TIMEOUT                 (1 << 10)
  89 #define ADD_STATS(elem, val)    do { (void)(val); } while(0)
  90
  91 static inline u64 spin_time_start(void)
  92 {
  93         return 0;
  94 }
  95
  96 static inline void spin_time_accum_fast(u64 start)
  97 {
  98 }
  99 static inline void spin_time_accum(u64 start)
 100 {
 101 }
 102 #endif  /* CONFIG_XEN_DEBUG_FS */
 103
 104 struct xen_spinlock {
 105         unsigned char lock;             /* 0 -> free; 1 -> locked */
 106         unsigned short spinners;        /* count of waiting cpus */
 107 };
 108
 109 static int xen_spin_is_locked(struct raw_spinlock *lock)
 110 {
 111         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 112
 113         return xl->lock != 0;
 114 }
 115
 116 static int xen_spin_is_contended(struct raw_spinlock *lock)
 117 {
 118         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 119
 120         /* Not strictly true; this is only the count of contended
 121            lock-takers entering the slow path. */
 122         return xl->spinners != 0;
 123 }
 124
 125 static int xen_spin_trylock(struct raw_spinlock *lock)
 126 {
 127         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 128         u8 old = 1;
 129
 130         asm("xchgb %b0,%1"
 131             : "+q" (old), "+m" (xl->lock) : : "memory");
 132
 133         return old == 0;
 134 }
 135
 136 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
 137 static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
 138
 139 /*
 140  * Mark a cpu as interested in a lock.  Returns the CPU's previous
 141  * lock of interest, in case we got preempted by an interrupt.
 142  */
 143 static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl)
 144 {
 145         struct xen_spinlock *prev;
 146
 147         prev = __get_cpu_var(lock_spinners);
 148         __get_cpu_var(lock_spinners) = xl;
 149
 150         wmb();                  /* set lock of interest before count */
 151
 152         asm(LOCK_PREFIX " incw %0"
 153             : "+m" (xl->spinners) : : "memory");
 154
 155         return prev;
 156 }
 157
 158 /*
 159  * Mark a cpu as no longer interested in a lock.  Restores previous
 160  * lock of interest (NULL for none).
 161  */
 162 static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev)
 163 {
 164         asm(LOCK_PREFIX " decw %0"
 165             : "+m" (xl->spinners) : : "memory");
 166         wmb();                  /* decrement count before restoring lock */
 167         __get_cpu_var(lock_spinners) = prev;
 168 }
 169
 170 static noinline int xen_spin_lock_slow(struct raw_spinlock *lock)
 171 {
 172         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 173         struct xen_spinlock *prev;
 174         int irq = __get_cpu_var(lock_kicker_irq);
 175         int ret;
 176
 177         /* If kicker interrupts not initialized yet, just spin */
 178         if (irq == -1)
 179                 return 0;
 180
 181         /* announce we're spinning */
 182         prev = spinning_lock(xl);
 183
 184         ADD_STATS(taken_slow, 1);
 185         ADD_STATS(taken_slow_nested, prev != NULL);
 186
 187         do {
 188                 /* clear pending */
 189                 xen_clear_irq_pending(irq);
 190
 191                 /* check again make sure it didn't become free while
 192                    we weren't looking  */
 193                 ret = xen_spin_trylock(lock);
 194                 if (ret) {
 195                         ADD_STATS(taken_slow_pickup, 1);
 196
 197                         /*
 198                          * If we interrupted another spinlock while it
 199                          * was blocking, make sure it doesn't block
 200                          * without rechecking the lock.
 201                          */
 202                         if (prev != NULL)
 203                                 xen_set_irq_pending(irq);
 204                         goto out;
 205                 }
 206
 207                 /*
 208                  * Block until irq becomes pending.  If we're
 209                  * interrupted at this point (after the trylock but
 210                  * before entering the block), then the nested lock
 211                  * handler guarantees that the irq will be left
 212                  * pending if there's any chance the lock became free;
 213                  * xen_poll_irq() returns immediately if the irq is
 214                  * pending.
 215                  */
 216                 xen_poll_irq(irq);
 217                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
 218         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
 219
 220         kstat_this_cpu.irqs[irq]++;
 221
 222 out:
 223         unspinning_lock(xl, prev);
 224         return ret;
 225 }
 226
 227 static void xen_spin_lock(struct raw_spinlock *lock)
 228 {
 229         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 230         unsigned timeout;
 231         u8 oldval;
 232         u64 start_spin;
 233
 234         ADD_STATS(taken, 1);
 235
 236         start_spin = spin_time_start();
 237
 238         do {
 239                 u64 start_spin_fast = spin_time_start();
 240
 241                 timeout = TIMEOUT;
 242
 243                 asm("1: xchgb %1,%0\n"
 244                     "   testb %1,%1\n"
 245                     "   jz 3f\n"
 246                     "2: rep;nop\n"
 247                     "   cmpb $0,%0\n"
 248                     "   je 1b\n"
 249                     "   dec %2\n"
 250                     "   jnz 2b\n"
 251                     "3:\n"
 252                     : "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
 253                     : "1" (1)
 254                     : "memory");
 255
 256                 spin_time_accum_fast(start_spin_fast);
 257         } while (unlikely(oldval != 0 && (TIMEOUT == ~0 || !xen_spin_lock_slow(lock))));
 258
 259         spin_time_accum(start_spin);
 260 }
 261
 262 static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
 263 {
 264         int cpu;
 265
 266         ADD_STATS(released_slow, 1);
 267
 268         for_each_online_cpu(cpu) {
 269                 /* XXX should mix up next cpu selection */
 270                 if (per_cpu(lock_spinners, cpu) == xl) {
 271                         ADD_STATS(released_slow_kicked, 1);
 272                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
 273                         break;
 274                 }
 275         }
 276 }
 277
 278 static void xen_spin_unlock(struct raw_spinlock *lock)
 279 {
 280         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
 281
 282         ADD_STATS(released, 1);
 283
 284         smp_wmb();              /* make sure no writes get moved after unlock */
 285         xl->lock = 0;           /* release lock */
 286
 287         /* make sure unlock happens before kick */
 288         barrier();
 289
 290         if (unlikely(xl->spinners))
 291                 xen_spin_unlock_slow(xl);
 292 }
 293
 294 static irqreturn_t dummy_handler(int irq, void *dev_id)
 295 {
 296         BUG();
 297         return IRQ_HANDLED;
 298 }
 299
 300 void __cpuinit xen_init_lock_cpu(int cpu)
 301 {
 302         int irq;
 303         const char *name;
 304
 305         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
 306         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
 307                                      cpu,
 308                                      dummy_handler,
 309                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 310                                      name,
 311                                      NULL);
 312
 313         if (irq >= 0) {
 314                 disable_irq(irq); /* make sure it's never delivered */
 315                 per_cpu(lock_kicker_irq, cpu) = irq;
 316         }
 317
 318         printk("cpu %d spinlock event irq %d\n", cpu, irq);
 319 }
 320
 321 void __init xen_init_spinlocks(void)
 322 {
 323         pv_lock_ops.spin_is_locked = xen_spin_is_locked;
 324         pv_lock_ops.spin_is_contended = xen_spin_is_contended;
 325         pv_lock_ops.spin_lock = xen_spin_lock;
 326         pv_lock_ops.spin_trylock = xen_spin_trylock;
 327         pv_lock_ops.spin_unlock = xen_spin_unlock;
 328 }
 329
 330 #ifdef CONFIG_XEN_DEBUG_FS
 331
 332 static struct dentry *d_spin_debug;
 333
 334 static int __init xen_spinlock_debugfs(void)
 335 {
 336         struct dentry *d_xen = xen_init_debugfs();
 337
 338         if (d_xen == NULL)
 339                 return -ENOMEM;
 340
 341         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
 342
 343         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
 344
 345         debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout);
 346
 347         debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken);
 348         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
 349                            &spinlock_stats.taken_slow);
 350         debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug,
 351                            &spinlock_stats.taken_slow_nested);
 352         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
 353                            &spinlock_stats.taken_slow_pickup);
 354         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
 355                            &spinlock_stats.taken_slow_spurious);
 356
 357         debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released);
 358         debugfs_create_u32("released_slow", 0444, d_spin_debug,
 359                            &spinlock_stats.released_slow);
 360         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
 361                            &spinlock_stats.released_slow_kicked);
 362
 363         debugfs_create_u64("time_spinning", 0444, d_spin_debug,
 364                            &spinlock_stats.spinning_time);
 365         debugfs_create_u64("time_total", 0444, d_spin_debug,
 366                            &spinlock_stats.total_time);
 367
 368         xen_debugfs_create_u32_array("histo_total", 0444, d_spin_debug,
 369                                      spinlock_stats.histo_spin, HISTO_BUCKETS + 1);
 370         xen_debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug,
 371                                      spinlock_stats.histo_spin_fast, HISTO_BUCKETS + 1);
 372
 373         return 0;
 374 }
 375 fs_initcall(xen_spinlock_debugfs);
 376
 377 #endif  /* CONFIG_XEN_DEBUG_FS */