arch/x86/oprofile/op_model_amd.c

   1 /*
   2  * @file op_model_amd.c
   3  * athlon / K7 / K8 / Family 10h model-specific MSR operations
   4  *
   5  * @remark Copyright 2002-2008 OProfile authors
   6  * @remark Read the file COPYING
   7  *
   8  * @author John Levon
   9  * @author Philippe Elie
  10  * @author Graydon Hoare
  11  * @author Robert Richter <robert.richter@amd.com>
  12  * @author Barry Kasindorf
  13 */
  14
  15 #include <linux/oprofile.h>
  16 #include <linux/device.h>
  17 #include <linux/pci.h>
  18 #include <linux/percpu.h>
  19
  20 #include <asm/ptrace.h>
  21 #include <asm/msr.h>
  22 #include <asm/nmi.h>
  23
  24 #include "op_x86_model.h"
  25 #include "op_counter.h"
  26
  27 #define NUM_COUNTERS 32
  28 #define NUM_HARDWARE_COUNTERS 4
  29 #define NUM_CONTROLS 32
  30 #define NUM_HARDWARE_CONTROLS 4
  31
  32 #define CTR_IS_RESERVED(msrs, c) (msrs->counters[(c)].addr ? 1 : 0)
  33 #define CTR_READ(l, h, msrs, c) do {rdmsr(msrs->counters[(c)].addr, (l), (h)); } while (0)
  34 #define CTR_WRITE(l, msrs, c) do {wrmsr(msrs->counters[(c)].addr, -(unsigned int)(l), -1); } while (0)
  35 #define CTR_OVERFLOWED(n) (!((n) & (1U<<31)))
  36
  37 #define CTRL_IS_RESERVED(msrs, c) (msrs->controls[(c)].addr ? 1 : 0)
  38 #define CTRL_READ(l, h, msrs, c) do {rdmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  39 #define CTRL_WRITE(l, h, msrs, c) do {wrmsr(msrs->controls[(c)].addr, (l), (h)); } while (0)
  40 #define CTRL_SET_ACTIVE(n) (n |= (1<<22))
  41 #define CTRL_SET_INACTIVE(n) (n &= ~(1<<22))
  42 #define CTRL_CLEAR_LO(x) (x &= (1<<21))
  43 #define CTRL_CLEAR_HI(x) (x &= 0xfffffcf0)
  44 #define CTRL_SET_ENABLE(val) (val |= 1<<20)
  45 #define CTRL_SET_USR(val, u) (val |= ((u & 1) << 16))
  46 #define CTRL_SET_KERN(val, k) (val |= ((k & 1) << 17))
  47 #define CTRL_SET_UM(val, m) (val |= (m << 8))
  48 #define CTRL_SET_EVENT_LOW(val, e) (val |= (e & 0xff))
  49 #define CTRL_SET_EVENT_HIGH(val, e) (val |= ((e >> 8) & 0xf))
  50 #define CTRL_SET_HOST_ONLY(val, h) (val |= ((h & 1) << 9))
  51 #define CTRL_SET_GUEST_ONLY(val, h) (val |= ((h & 1) << 8))
  52
  53 static unsigned long reset_value[NUM_COUNTERS];
  54 DECLARE_PER_CPU(int, switch_index);
  55
  56 #ifdef CONFIG_OPROFILE_IBS
  57
  58 /* IbsFetchCtl bits/masks */
  59 #define IBS_FETCH_HIGH_VALID_BIT        (1UL << 17)     /* bit 49 */
  60 #define IBS_FETCH_HIGH_ENABLE           (1UL << 16)     /* bit 48 */
  61 #define IBS_FETCH_LOW_MAX_CNT_MASK      0x0000FFFFUL    /* MaxCnt mask */
  62
  63 /*IbsOpCtl bits */
  64 #define IBS_OP_LOW_VALID_BIT            (1ULL<<18)      /* bit 18 */
  65 #define IBS_OP_LOW_ENABLE               (1ULL<<17)      /* bit 17 */
  66
  67 /* Codes used in cpu_buffer.c */
  68 /* This produces duplicate code, need to be fixed */
  69 #define IBS_FETCH_BEGIN 3
  70 #define IBS_OP_BEGIN    4
  71
  72 /* The function interface needs to be fixed, something like add
  73    data. Should then be added to linux/oprofile.h. */
  74 extern void oprofile_add_ibs_sample(struct pt_regs *const regs,
  75                                     unsigned int * const ibs_sample, u8 code);
  76
  77 struct ibs_fetch_sample {
  78         /* MSRC001_1031 IBS Fetch Linear Address Register */
  79         unsigned int ibs_fetch_lin_addr_low;
  80         unsigned int ibs_fetch_lin_addr_high;
  81         /* MSRC001_1030 IBS Fetch Control Register */
  82         unsigned int ibs_fetch_ctl_low;
  83         unsigned int ibs_fetch_ctl_high;
  84         /* MSRC001_1032 IBS Fetch Physical Address Register */
  85         unsigned int ibs_fetch_phys_addr_low;
  86         unsigned int ibs_fetch_phys_addr_high;
  87 };
  88
  89 struct ibs_op_sample {
  90         /* MSRC001_1034 IBS Op Logical Address Register (IbsRIP) */
  91         unsigned int ibs_op_rip_low;
  92         unsigned int ibs_op_rip_high;
  93         /* MSRC001_1035 IBS Op Data Register */
  94         unsigned int ibs_op_data1_low;
  95         unsigned int ibs_op_data1_high;
  96         /* MSRC001_1036 IBS Op Data 2 Register */
  97         unsigned int ibs_op_data2_low;
  98         unsigned int ibs_op_data2_high;
  99         /* MSRC001_1037 IBS Op Data 3 Register */
 100         unsigned int ibs_op_data3_low;
 101         unsigned int ibs_op_data3_high;
 102         /* MSRC001_1038 IBS DC Linear Address Register (IbsDcLinAd) */
 103         unsigned int ibs_dc_linear_low;
 104         unsigned int ibs_dc_linear_high;
 105         /* MSRC001_1039 IBS DC Physical Address Register (IbsDcPhysAd) */
 106         unsigned int ibs_dc_phys_low;
 107         unsigned int ibs_dc_phys_high;
 108 };
 109
 110 /*
 111  * unitialize the APIC for the IBS interrupts if needed on AMD Family10h+
 112 */
 113 static void clear_ibs_nmi(void);
 114
 115 static int ibs_allowed; /* AMD Family10h and later */
 116
 117 struct op_ibs_config {
 118         unsigned long op_enabled;
 119         unsigned long fetch_enabled;
 120         unsigned long max_cnt_fetch;
 121         unsigned long max_cnt_op;
 122         unsigned long rand_en;
 123         unsigned long dispatched_ops;
 124 };
 125
 126 static struct op_ibs_config ibs_config;
 127
 128 #endif
 129
 130 /* functions for op_amd_spec */
 131
 132 static void op_amd_fill_in_addresses(struct op_msrs * const msrs)
 133 {
 134         int i;
 135
 136         for (i = 0; i < NUM_COUNTERS; i++) {
 137                 int hw_counter = i % NUM_HARDWARE_COUNTERS;
 138                 if (reserve_perfctr_nmi(MSR_K7_PERFCTR0 + hw_counter))
 139                         msrs->counters[i].addr = MSR_K7_PERFCTR0 + hw_counter;
 140                 else
 141                         msrs->counters[i].addr = 0;
 142         }
 143
 144         for (i = 0; i < NUM_CONTROLS; i++) {
 145                 int hw_control = i % NUM_HARDWARE_CONTROLS;
 146                 if (reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + hw_control))
 147                         msrs->controls[i].addr = MSR_K7_EVNTSEL0 + hw_control;
 148                 else
 149                         msrs->controls[i].addr = 0;
 150         }
 151 }
 152
 153
 154 static void op_amd_setup_ctrs(struct op_msrs const * const msrs)
 155 {
 156         unsigned int low, high;
 157         int i;
 158
 159         for (i = 0; i < NUM_HARDWARE_CONTROLS; ++i) {
 160                 int offset = i + __get_cpu_var(switch_index);
 161                 if (counter_config[offset].enabled)
 162                         reset_value[offset] = counter_config[offset].count;
 163                 else
 164                         reset_value[offset] = 0;
 165         }
 166
 167         /* clear all counters */
 168         for (i = 0 ; i < NUM_HARDWARE_CONTROLS; ++i) {
 169                 if (unlikely(!CTRL_IS_RESERVED(msrs, i)))
 170                         continue;
 171                 CTRL_READ(low, high, msrs, i);
 172                 CTRL_CLEAR_LO(low);
 173                 CTRL_CLEAR_HI(high);
 174                 CTRL_WRITE(low, high, msrs, i);
 175         }
 176
 177         /* avoid a false detection of ctr overflows in NMI handler */
 178         for (i = 0; i < NUM_HARDWARE_COUNTERS; ++i) {
 179                 if (unlikely(!CTR_IS_RESERVED(msrs, i)))
 180                         continue;
 181                 CTR_WRITE(1, msrs, i);
 182         }
 183
 184         /* enable active counters */
 185         for (i = 0; i < NUM_HARDWARE_COUNTERS; ++i) {
 186                 int offset = i + __get_cpu_var(switch_index);
 187                 if ((counter_config[offset].enabled) && (CTR_IS_RESERVED(msrs, i))) {
 188                         CTR_WRITE(counter_config[offset].count, msrs, i);
 189
 190                         CTRL_READ(low, high, msrs, i);
 191                         CTRL_CLEAR_LO(low);
 192                         CTRL_CLEAR_HI(high);
 193                         CTRL_SET_ENABLE(low);
 194                         CTRL_SET_USR(low, counter_config[offset].user);
 195                         CTRL_SET_KERN(low, counter_config[offset].kernel);
 196                         CTRL_SET_UM(low, counter_config[offset].unit_mask);
 197                         CTRL_SET_EVENT_LOW(low, counter_config[offset].event);
 198                         CTRL_SET_EVENT_HIGH(high, counter_config[offset].event);
 199                         CTRL_SET_HOST_ONLY(high, 0);
 200                         CTRL_SET_GUEST_ONLY(high, 0);
 201
 202                         CTRL_WRITE(low, high, msrs, i);
 203                 }
 204         }
 205 }
 206
 207 #ifdef CONFIG_OPROFILE_IBS
 208
 209 static inline int
 210 op_amd_handle_ibs(struct pt_regs * const regs,
 211                   struct op_msrs const * const msrs)
 212 {
 213         unsigned int low, high;
 214         struct ibs_fetch_sample ibs_fetch;
 215         struct ibs_op_sample ibs_op;
 216
 217         if (!ibs_allowed)
 218                 return 1;
 219
 220         if (ibs_config.fetch_enabled) {
 221                 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 222                 if (high & IBS_FETCH_HIGH_VALID_BIT) {
 223                         ibs_fetch.ibs_fetch_ctl_high = high;
 224                         ibs_fetch.ibs_fetch_ctl_low = low;
 225                         rdmsr(MSR_AMD64_IBSFETCHLINAD, low, high);
 226                         ibs_fetch.ibs_fetch_lin_addr_high = high;
 227                         ibs_fetch.ibs_fetch_lin_addr_low = low;
 228                         rdmsr(MSR_AMD64_IBSFETCHPHYSAD, low, high);
 229                         ibs_fetch.ibs_fetch_phys_addr_high = high;
 230                         ibs_fetch.ibs_fetch_phys_addr_low = low;
 231
 232                         oprofile_add_ibs_sample(regs,
 233                                                 (unsigned int *)&ibs_fetch,
 234                                                 IBS_FETCH_BEGIN);
 235
 236                         /*reenable the IRQ */
 237                         rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 238                         high &= ~IBS_FETCH_HIGH_VALID_BIT;
 239                         high |= IBS_FETCH_HIGH_ENABLE;
 240                         low &= IBS_FETCH_LOW_MAX_CNT_MASK;
 241                         wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 242                 }
 243         }
 244
 245         if (ibs_config.op_enabled) {
 246                 rdmsr(MSR_AMD64_IBSOPCTL, low, high);
 247                 if (low & IBS_OP_LOW_VALID_BIT) {
 248                         rdmsr(MSR_AMD64_IBSOPRIP, low, high);
 249                         ibs_op.ibs_op_rip_low = low;
 250                         ibs_op.ibs_op_rip_high = high;
 251                         rdmsr(MSR_AMD64_IBSOPDATA, low, high);
 252                         ibs_op.ibs_op_data1_low = low;
 253                         ibs_op.ibs_op_data1_high = high;
 254                         rdmsr(MSR_AMD64_IBSOPDATA2, low, high);
 255                         ibs_op.ibs_op_data2_low = low;
 256                         ibs_op.ibs_op_data2_high = high;
 257                         rdmsr(MSR_AMD64_IBSOPDATA3, low, high);
 258                         ibs_op.ibs_op_data3_low = low;
 259                         ibs_op.ibs_op_data3_high = high;
 260                         rdmsr(MSR_AMD64_IBSDCLINAD, low, high);
 261                         ibs_op.ibs_dc_linear_low = low;
 262                         ibs_op.ibs_dc_linear_high = high;
 263                         rdmsr(MSR_AMD64_IBSDCPHYSAD, low, high);
 264                         ibs_op.ibs_dc_phys_low = low;
 265                         ibs_op.ibs_dc_phys_high = high;
 266
 267                         /* reenable the IRQ */
 268                         oprofile_add_ibs_sample(regs,
 269                                                 (unsigned int *)&ibs_op,
 270                                                 IBS_OP_BEGIN);
 271                         rdmsr(MSR_AMD64_IBSOPCTL, low, high);
 272                         high = 0;
 273                         low &= ~IBS_OP_LOW_VALID_BIT;
 274                         low |= IBS_OP_LOW_ENABLE;
 275                         wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 276                 }
 277         }
 278
 279         return 1;
 280 }
 281
 282 #endif
 283
 284 static int op_amd_check_ctrs(struct pt_regs * const regs,
 285                              struct op_msrs const * const msrs)
 286 {
 287         unsigned int low, high;
 288         int i;
 289
 290         for (i = 0 ; i < NUM_HARDWARE_COUNTERS ; ++i) {
 291                 int offset = i + __get_cpu_var(switch_index);
 292                 if (!reset_value[offset])
 293                         continue;
 294                 CTR_READ(low, high, msrs, i);
 295                 if (CTR_OVERFLOWED(low)) {
 296                         oprofile_add_sample(regs, offset);
 297                         CTR_WRITE(reset_value[offset], msrs, i);
 298                 }
 299         }
 300
 301 #ifdef CONFIG_OPROFILE_IBS
 302         op_amd_handle_ibs(regs, msrs);
 303 #endif
 304
 305         /* See op_model_ppro.c */
 306         return 1;
 307 }
 308
 309 static void op_amd_start(struct op_msrs const * const msrs)
 310 {
 311         unsigned int low, high;
 312         int i;
 313
 314         for (i = 0 ; i < NUM_HARDWARE_COUNTERS ; ++i) {
 315                 int offset = i + __get_cpu_var(switch_index);
 316                 if (reset_value[offset]) {
 317                         CTRL_READ(low, high, msrs, i);
 318                         CTRL_SET_ACTIVE(low);
 319                         CTRL_WRITE(low, high, msrs, i);
 320                 }
 321         }
 322
 323 #ifdef CONFIG_OPROFILE_IBS
 324         if (ibs_allowed && ibs_config.fetch_enabled) {
 325                 low = (ibs_config.max_cnt_fetch >> 4) & 0xFFFF;
 326                 high = IBS_FETCH_HIGH_ENABLE;
 327                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 328         }
 329
 330         if (ibs_allowed && ibs_config.op_enabled) {
 331                 low = ((ibs_config.max_cnt_op >> 4) & 0xFFFF) + IBS_OP_LOW_ENABLE;
 332                 high = 0;
 333                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 334         }
 335 #endif
 336 }
 337
 338
 339 static void op_amd_stop(struct op_msrs const * const msrs)
 340 {
 341         unsigned int low, high;
 342         int i;
 343
 344         /* Subtle: stop on all counters to avoid race with
 345          * setting our pm callback */
 346         for (i = 0 ; i < NUM_HARDWARE_COUNTERS ; ++i) {
 347                 if (!reset_value[i + per_cpu(switch_index, smp_processor_id())])
 348                         continue;
 349                 CTRL_READ(low, high, msrs, i);
 350                 CTRL_SET_INACTIVE(low);
 351                 CTRL_WRITE(low, high, msrs, i);
 352         }
 353
 354 #ifdef CONFIG_OPROFILE_IBS
 355         if (ibs_allowed && ibs_config.fetch_enabled) {
 356                 low = 0;                /* clear max count and enable */
 357                 high = 0;
 358                 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
 359         }
 360
 361         if (ibs_allowed && ibs_config.op_enabled) {
 362                 low = 0;                /* clear max count and enable */
 363                 high = 0;
 364                 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
 365         }
 366 #endif
 367 }
 368
 369 static void op_amd_shutdown(struct op_msrs const * const msrs)
 370 {
 371         int i;
 372
 373         for (i = 0 ; i < NUM_HARDWARE_COUNTERS ; ++i) {
 374                 if (CTR_IS_RESERVED(msrs, i))
 375                         release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
 376         }
 377         for (i = 0 ; i < NUM_HARDWARE_COUNTERS ; ++i) {
 378                 if (CTRL_IS_RESERVED(msrs, i))
 379                         release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
 380         }
 381 }
 382
 383 #ifndef CONFIG_OPROFILE_IBS
 384
 385 /* no IBS support */
 386
 387 static int op_amd_init(struct oprofile_operations *ops)
 388 {
 389         return 0;
 390 }
 391
 392 static void op_amd_exit(void) {}
 393
 394 #else
 395
 396 static u8 ibs_eilvt_off;
 397
 398 static inline void apic_init_ibs_nmi_per_cpu(void *arg)
 399 {
 400         ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
 401 }
 402
 403 static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
 404 {
 405         setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 406 }
 407
 408 static int pfm_amd64_setup_eilvt(void)
 409 {
 410 #define IBSCTL_LVTOFFSETVAL             (1 << 8)
 411 #define IBSCTL                          0x1cc
 412         struct pci_dev *cpu_cfg;
 413         int nodes;
 414         u32 value = 0;
 415
 416         /* per CPU setup */
 417         on_each_cpu(apic_init_ibs_nmi_per_cpu, NULL, 1);
 418
 419         nodes = 0;
 420         cpu_cfg = NULL;
 421         do {
 422                 cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD,
 423                                          PCI_DEVICE_ID_AMD_10H_NB_MISC,
 424                                          cpu_cfg);
 425                 if (!cpu_cfg)
 426                         break;
 427                 ++nodes;
 428                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
 429                                        | IBSCTL_LVTOFFSETVAL);
 430                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
 431                 if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
 432                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
 433                                 "IBSCTL = 0x%08x", value);
 434                         return 1;
 435                 }
 436         } while (1);
 437
 438         if (!nodes) {
 439                 printk(KERN_DEBUG "No CPU node configured for IBS");
 440                 return 1;
 441         }
 442
 443 #ifdef CONFIG_NUMA
 444         /* Sanity check */
 445         /* Works only for 64bit with proper numa implementation. */
 446         if (nodes != num_possible_nodes()) {
 447                 printk(KERN_DEBUG "Failed to setup CPU node(s) for IBS, "
 448                         "found: %d, expected %d",
 449                         nodes, num_possible_nodes());
 450                 return 1;
 451         }
 452 #endif
 453         return 0;
 454 }
 455
 456 /*
 457  * initialize the APIC for the IBS interrupts
 458  * if available (AMD Family10h rev B0 and later)
 459  */
 460 static void setup_ibs(void)
 461 {
 462         ibs_allowed = boot_cpu_has(X86_FEATURE_IBS);
 463
 464         if (!ibs_allowed)
 465                 return;
 466
 467         if (pfm_amd64_setup_eilvt()) {
 468                 ibs_allowed = 0;
 469                 return;
 470         }
 471
 472         printk(KERN_INFO "oprofile: AMD IBS detected\n");
 473 }
 474
 475
 476 /*
 477  * unitialize the APIC for the IBS interrupts if needed on AMD Family10h
 478  * rev B0 and later */
 479 static void clear_ibs_nmi(void)
 480 {
 481         if (ibs_allowed)
 482                 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
 483 }
 484
 485 static int (*create_arch_files)(struct super_block * sb, struct dentry * root);
 486
 487 static int setup_ibs_files(struct super_block * sb, struct dentry * root)
 488 {
 489         char buf[12];
 490         struct dentry *dir;
 491         int ret = 0;
 492
 493         /* architecture specific files */
 494         if (create_arch_files)
 495                 ret = create_arch_files(sb, root);
 496
 497         if (ret)
 498                 return ret;
 499
 500         if (!ibs_allowed)
 501                 return ret;
 502
 503         /* model specific files */
 504
 505         /* setup some reasonable defaults */
 506         ibs_config.max_cnt_fetch = 250000;
 507         ibs_config.fetch_enabled = 0;
 508         ibs_config.max_cnt_op = 250000;
 509         ibs_config.op_enabled = 0;
 510         ibs_config.dispatched_ops = 1;
 511         snprintf(buf,  sizeof(buf), "ibs_fetch");
 512         dir = oprofilefs_mkdir(sb, root, buf);
 513         oprofilefs_create_ulong(sb, dir, "rand_enable",
 514                                 &ibs_config.rand_en);
 515         oprofilefs_create_ulong(sb, dir, "enable",
 516                 &ibs_config.fetch_enabled);
 517         oprofilefs_create_ulong(sb, dir, "max_count",
 518                 &ibs_config.max_cnt_fetch);
 519         snprintf(buf,  sizeof(buf), "ibs_uops");
 520         dir = oprofilefs_mkdir(sb, root, buf);
 521         oprofilefs_create_ulong(sb, dir, "enable",
 522                 &ibs_config.op_enabled);
 523         oprofilefs_create_ulong(sb, dir, "max_count",
 524                 &ibs_config.max_cnt_op);
 525         oprofilefs_create_ulong(sb, dir, "dispatched_ops",
 526                 &ibs_config.dispatched_ops);
 527
 528         return 0;
 529 }
 530
 531 static int op_amd_init(struct oprofile_operations *ops)
 532 {
 533         setup_ibs();
 534         create_arch_files = ops->create_files;
 535         ops->create_files = setup_ibs_files;
 536         return 0;
 537 }
 538
 539 static void op_amd_exit(void)
 540 {
 541         clear_ibs_nmi();
 542 }
 543
 544 #endif
 545
 546 struct op_x86_model_spec const op_amd_spec = {
 547         .init = op_amd_init,
 548         .exit = op_amd_exit,
 549         .num_counters = NUM_COUNTERS,
 550         .num_controls = NUM_CONTROLS,
 551         .num_hardware_counters = NUM_HARDWARE_COUNTERS,
 552         .num_hardware_controls = NUM_HARDWARE_CONTROLS,
 553         .fill_in_addresses = &op_amd_fill_in_addresses,
 554         .setup_ctrs = &op_amd_setup_ctrs,
 555         .check_ctrs = &op_amd_check_ctrs,
 556         .start = &op_amd_start,
 557         .stop = &op_amd_stop,
 558         .shutdown = &op_amd_shutdown
 559 };