arch/x86/xen/smp.c

   1 /*
   2  * Xen SMP support
   3  *
   4  * This file implements the Xen versions of smp_ops.  SMP under Xen is
   5  * very straightforward.  Bringing a CPU up is simply a matter of
   6  * loading its initial context and setting it running.
   7  *
   8  * IPIs are handled through the Xen event mechanism.
   9  *
  10  * Because virtual CPUs can be scheduled onto any real CPU, there's no
  11  * useful topology information for the kernel to make use of.  As a
  12  * result, all CPUs are treated as if they're single-core and
  13  * single-threaded.
  14  *
  15  * This does not handle HOTPLUG_CPU yet.
  16  */
  17 #include <linux/sched.h>
  18 #include <linux/err.h>
  19 #include <linux/smp.h>
  20
  21 #include <asm/paravirt.h>
  22 #include <asm/desc.h>
  23 #include <asm/pgtable.h>
  24 #include <asm/cpu.h>
  25
  26 #include <xen/interface/xen.h>
  27 #include <xen/interface/vcpu.h>
  28
  29 #include <asm/xen/interface.h>
  30 #include <asm/xen/hypercall.h>
  31
  32 #include <xen/page.h>
  33 #include <xen/events.h>
  34
  35 #include "xen-ops.h"
  36 #include "mmu.h"
  37
  38 cpumask_t xen_cpu_initialized_map;
  39
  40 static DEFINE_PER_CPU(int, resched_irq);
  41 static DEFINE_PER_CPU(int, callfunc_irq);
  42 static DEFINE_PER_CPU(int, callfuncsingle_irq);
  43 static DEFINE_PER_CPU(int, debug_irq) = -1;
  44
  45 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
  46 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
  47
  48 /*
  49  * Reschedule call back. Nothing to do,
  50  * all the work is done automatically when
  51  * we return from the interrupt.
  52  */
  53 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
  54 {
  55 #ifdef CONFIG_X86_32
  56         __get_cpu_var(irq_stat).irq_resched_count++;
  57 #else
  58         add_pda(irq_resched_count, 1);
  59 #endif
  60
  61         return IRQ_HANDLED;
  62 }
  63
  64 static __cpuinit void cpu_bringup_and_idle(void)
  65 {
  66         int cpu = smp_processor_id();
  67
  68         cpu_init();
  69         preempt_disable();
  70
  71         xen_enable_sysenter();
  72
  73         cpu = smp_processor_id();
  74         smp_store_cpu_info(cpu);
  75         cpu_data(cpu).x86_max_cores = 1;
  76         set_cpu_sibling_map(cpu);
  77
  78         xen_setup_cpu_clockevents();
  79
  80         cpu_set(cpu, cpu_online_map);
  81         x86_write_percpu(cpu_state, CPU_ONLINE);
  82         wmb();
  83
  84         /* We can take interrupts now: we're officially "up". */
  85         local_irq_enable();
  86
  87         wmb();                  /* make sure everything is out */
  88         cpu_idle();
  89 }
  90
  91 static int xen_smp_intr_init(unsigned int cpu)
  92 {
  93         int rc;
  94         const char *resched_name, *callfunc_name, *debug_name;
  95
  96         resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
  97         rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
  98                                     cpu,
  99                                     xen_reschedule_interrupt,
 100                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 101                                     resched_name,
 102                                     NULL);
 103         if (rc < 0)
 104                 goto fail;
 105         per_cpu(resched_irq, cpu) = rc;
 106
 107         callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
 108         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
 109                                     cpu,
 110                                     xen_call_function_interrupt,
 111                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 112                                     callfunc_name,
 113                                     NULL);
 114         if (rc < 0)
 115                 goto fail;
 116         per_cpu(callfunc_irq, cpu) = rc;
 117
 118         debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
 119         rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu, xen_debug_interrupt,
 120                                      IRQF_DISABLED | IRQF_PERCPU | IRQF_NOBALANCING,
 121                                      debug_name, NULL);
 122         if (rc < 0)
 123                 goto fail;
 124         per_cpu(debug_irq, cpu) = rc;
 125
 126         callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
 127         rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
 128                                     cpu,
 129                                     xen_call_function_single_interrupt,
 130                                     IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 131                                     callfunc_name,
 132                                     NULL);
 133         if (rc < 0)
 134                 goto fail;
 135         per_cpu(callfuncsingle_irq, cpu) = rc;
 136
 137         return 0;
 138
 139  fail:
 140         if (per_cpu(resched_irq, cpu) >= 0)
 141                 unbind_from_irqhandler(per_cpu(resched_irq, cpu), NULL);
 142         if (per_cpu(callfunc_irq, cpu) >= 0)
 143                 unbind_from_irqhandler(per_cpu(callfunc_irq, cpu), NULL);
 144         if (per_cpu(debug_irq, cpu) >= 0)
 145                 unbind_from_irqhandler(per_cpu(debug_irq, cpu), NULL);
 146         if (per_cpu(callfuncsingle_irq, cpu) >= 0)
 147                 unbind_from_irqhandler(per_cpu(callfuncsingle_irq, cpu), NULL);
 148
 149         return rc;
 150 }
 151
 152 static void __init xen_fill_possible_map(void)
 153 {
 154         int i, rc;
 155
 156         for (i = 0; i < NR_CPUS; i++) {
 157                 rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
 158                 if (rc >= 0)
 159                         cpu_set(i, cpu_possible_map);
 160         }
 161 }
 162
 163 static void __init xen_smp_prepare_boot_cpu(void)
 164 {
 165         BUG_ON(smp_processor_id() != 0);
 166         native_smp_prepare_boot_cpu();
 167
 168         /* We've switched to the "real" per-cpu gdt, so make sure the
 169            old memory can be recycled */
 170         make_lowmem_page_readwrite(&per_cpu_var(gdt_page));
 171
 172         xen_setup_vcpu_info_placement();
 173 }
 174
 175 static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
 176 {
 177         unsigned cpu;
 178
 179         smp_store_cpu_info(0);
 180         cpu_data(0).x86_max_cores = 1;
 181         set_cpu_sibling_map(0);
 182
 183         if (xen_smp_intr_init(0))
 184                 BUG();
 185
 186         xen_cpu_initialized_map = cpumask_of_cpu(0);
 187
 188         /* Restrict the possible_map according to max_cpus. */
 189         while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
 190                 for (cpu = NR_CPUS - 1; !cpu_possible(cpu); cpu--)
 191                         continue;
 192                 cpu_clear(cpu, cpu_possible_map);
 193         }
 194
 195         for_each_possible_cpu (cpu) {
 196                 struct task_struct *idle;
 197
 198                 if (cpu == 0)
 199                         continue;
 200
 201                 idle = fork_idle(cpu);
 202                 if (IS_ERR(idle))
 203                         panic("failed fork for CPU %d", cpu);
 204
 205                 cpu_set(cpu, cpu_present_map);
 206         }
 207
 208         //init_xenbus_allowed_cpumask();
 209 }
 210
 211 static __cpuinit int
 212 cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 213 {
 214         struct vcpu_guest_context *ctxt;
 215         struct desc_struct *gdt;
 216
 217         if (cpu_test_and_set(cpu, xen_cpu_initialized_map))
 218                 return 0;
 219
 220         ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
 221         if (ctxt == NULL)
 222                 return -ENOMEM;
 223
 224         gdt = get_cpu_gdt_table(cpu);
 225
 226         ctxt->flags = VGCF_IN_KERNEL;
 227         ctxt->user_regs.ds = __USER_DS;
 228         ctxt->user_regs.es = __USER_DS;
 229         ctxt->user_regs.ss = __KERNEL_DS;
 230 #ifdef CONFIG_X86_32
 231         ctxt->user_regs.fs = __KERNEL_PERCPU;
 232 #endif
 233         ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
 234         ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
 235
 236         memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
 237
 238         xen_copy_trap_info(ctxt->trap_ctxt);
 239
 240         ctxt->ldt_ents = 0;
 241
 242         BUG_ON((unsigned long)gdt & ~PAGE_MASK);
 243         make_lowmem_page_readonly(gdt);
 244
 245         ctxt->gdt_frames[0] = virt_to_mfn(gdt);
 246         ctxt->gdt_ents      = GDT_ENTRIES;
 247
 248         ctxt->user_regs.cs = __KERNEL_CS;
 249         ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
 250
 251         ctxt->kernel_ss = __KERNEL_DS;
 252         ctxt->kernel_sp = idle->thread.sp0;
 253
 254 #ifdef CONFIG_X86_32
 255         ctxt->event_callback_cs     = __KERNEL_CS;
 256         ctxt->failsafe_callback_cs  = __KERNEL_CS;
 257 #endif
 258         ctxt->event_callback_eip    = (unsigned long)xen_hypervisor_callback;
 259         ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback;
 260
 261         per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
 262         ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_mfn(swapper_pg_dir));
 263
 264         if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
 265                 BUG();
 266
 267         kfree(ctxt);
 268         return 0;
 269 }
 270
 271 static int __cpuinit xen_cpu_up(unsigned int cpu)
 272 {
 273         struct task_struct *idle = idle_task(cpu);
 274         int rc;
 275
 276 #if 0
 277         rc = cpu_up_check(cpu);
 278         if (rc)
 279                 return rc;
 280 #endif
 281
 282 #ifdef CONFIG_X86_64
 283         /* Allocate node local memory for AP pdas */
 284         WARN_ON(cpu == 0);
 285         if (cpu > 0) {
 286                 rc = get_local_pda(cpu);
 287                 if (rc)
 288                         return rc;
 289         }
 290 #endif
 291
 292 #ifdef CONFIG_X86_32
 293         init_gdt(cpu);
 294         per_cpu(current_task, cpu) = idle;
 295         irq_ctx_init(cpu);
 296 #else
 297         cpu_pda(cpu)->pcurrent = idle;
 298         clear_tsk_thread_flag(idle, TIF_FORK);
 299 #endif
 300         xen_setup_timer(cpu);
 301
 302         per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
 303
 304         /* make sure interrupts start blocked */
 305         per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
 306
 307         rc = cpu_initialize_context(cpu, idle);
 308         if (rc)
 309                 return rc;
 310
 311         if (num_online_cpus() == 1)
 312                 alternatives_smp_switch(1);
 313
 314         rc = xen_smp_intr_init(cpu);
 315         if (rc)
 316                 return rc;
 317
 318         rc = HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
 319         BUG_ON(rc);
 320
 321         while(per_cpu(cpu_state, cpu) != CPU_ONLINE) {
 322                 HYPERVISOR_sched_op(SCHEDOP_yield, 0);
 323                 barrier();
 324         }
 325
 326         return 0;
 327 }
 328
 329 static void xen_smp_cpus_done(unsigned int max_cpus)
 330 {
 331 }
 332
 333 static void stop_self(void *v)
 334 {
 335         int cpu = smp_processor_id();
 336
 337         /* make sure we're not pinning something down */
 338         load_cr3(swapper_pg_dir);
 339         /* should set up a minimal gdt */
 340
 341         HYPERVISOR_vcpu_op(VCPUOP_down, cpu, NULL);
 342         BUG();
 343 }
 344
 345 static void xen_smp_send_stop(void)
 346 {
 347         smp_call_function(stop_self, NULL, 0);
 348 }
 349
 350 static void xen_smp_send_reschedule(int cpu)
 351 {
 352         xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
 353 }
 354
 355 static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector)
 356 {
 357         unsigned cpu;
 358
 359         cpus_and(mask, mask, cpu_online_map);
 360
 361         for_each_cpu_mask(cpu, mask)
 362                 xen_send_IPI_one(cpu, vector);
 363 }
 364
 365 static void xen_smp_send_call_function_ipi(cpumask_t mask)
 366 {
 367         int cpu;
 368
 369         xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
 370
 371         /* Make sure other vcpus get a chance to run if they need to. */
 372         for_each_cpu_mask(cpu, mask) {
 373                 if (xen_vcpu_stolen(cpu)) {
 374                         HYPERVISOR_sched_op(SCHEDOP_yield, 0);
 375                         break;
 376                 }
 377         }
 378 }
 379
 380 static void xen_smp_send_call_function_single_ipi(int cpu)
 381 {
 382         xen_send_IPI_mask(cpumask_of_cpu(cpu), XEN_CALL_FUNCTION_SINGLE_VECTOR);
 383 }
 384
 385 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
 386 {
 387         irq_enter();
 388         generic_smp_call_function_interrupt();
 389 #ifdef CONFIG_X86_32
 390         __get_cpu_var(irq_stat).irq_call_count++;
 391 #else
 392         add_pda(irq_call_count, 1);
 393 #endif
 394         irq_exit();
 395
 396         return IRQ_HANDLED;
 397 }
 398
 399 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
 400 {
 401         irq_enter();
 402         generic_smp_call_function_single_interrupt();
 403 #ifdef CONFIG_X86_32
 404         __get_cpu_var(irq_stat).irq_call_count++;
 405 #else
 406         add_pda(irq_call_count, 1);
 407 #endif
 408         irq_exit();
 409
 410         return IRQ_HANDLED;
 411 }
 412
 413 static const struct smp_ops xen_smp_ops __initdata = {
 414         .smp_prepare_boot_cpu = xen_smp_prepare_boot_cpu,
 415         .smp_prepare_cpus = xen_smp_prepare_cpus,
 416         .cpu_up = xen_cpu_up,
 417         .smp_cpus_done = xen_smp_cpus_done,
 418
 419         .smp_send_stop = xen_smp_send_stop,
 420         .smp_send_reschedule = xen_smp_send_reschedule,
 421
 422         .send_call_func_ipi = xen_smp_send_call_function_ipi,
 423         .send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
 424 };
 425
 426 void __init xen_smp_init(void)
 427 {
 428         smp_ops = xen_smp_ops;
 429         xen_fill_possible_map();
 430 }