drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34
  35 #undef PREFIX
  36 #define PREFIX "DMAR:"
  37
  38 /* No locks are needed as DMA remapping hardware unit
  39  * list is constructed at boot time and hotplug of
  40  * these units are not supported by the architecture.
  41  */
  42 LIST_HEAD(dmar_drhd_units);
  43
  44 static struct acpi_table_header * __initdata dmar_tbl;
  45
  46 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  47 {
  48         /*
  49          * add INCLUDE_ALL at the tail, so scan the list will find it at
  50          * the very end.
  51          */
  52         if (drhd->include_all)
  53                 list_add_tail(&drhd->list, &dmar_drhd_units);
  54         else
  55                 list_add(&drhd->list, &dmar_drhd_units);
  56 }
  57
  58 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  59                                            struct pci_dev **dev, u16 segment)
  60 {
  61         struct pci_bus *bus;
  62         struct pci_dev *pdev = NULL;
  63         struct acpi_dmar_pci_path *path;
  64         int count;
  65
  66         bus = pci_find_bus(segment, scope->bus);
  67         path = (struct acpi_dmar_pci_path *)(scope + 1);
  68         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  69                 / sizeof(struct acpi_dmar_pci_path);
  70
  71         while (count) {
  72                 if (pdev)
  73                         pci_dev_put(pdev);
  74                 /*
  75                  * Some BIOSes list non-exist devices in DMAR table, just
  76                  * ignore it
  77                  */
  78                 if (!bus) {
  79                         printk(KERN_WARNING
  80                         PREFIX "Device scope bus [%d] not found\n",
  81                         scope->bus);
  82                         break;
  83                 }
  84                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  85                 if (!pdev) {
  86                         printk(KERN_WARNING PREFIX
  87                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  88                                 segment, bus->number, path->dev, path->fn);
  89                         break;
  90                 }
  91                 path ++;
  92                 count --;
  93                 bus = pdev->subordinate;
  94         }
  95         if (!pdev) {
  96                 printk(KERN_WARNING PREFIX
  97                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  98                 segment, scope->bus, path->dev, path->fn);
  99                 *dev = NULL;
 100                 return 0;
 101         }
 102         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 103                         pdev->subordinate) || (scope->entry_type == \
 104                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 105                 pci_dev_put(pdev);
 106                 printk(KERN_WARNING PREFIX
 107                         "Device scope type does not match for %s\n",
 108                          pci_name(pdev));
 109                 return -EINVAL;
 110         }
 111         *dev = pdev;
 112         return 0;
 113 }
 114
 115 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 116                                        struct pci_dev ***devices, u16 segment)
 117 {
 118         struct acpi_dmar_device_scope *scope;
 119         void * tmp = start;
 120         int index;
 121         int ret;
 122
 123         *cnt = 0;
 124         while (start < end) {
 125                 scope = start;
 126                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 127                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 128                         (*cnt)++;
 129                 else
 130                         printk(KERN_WARNING PREFIX
 131                                 "Unsupported device scope\n");
 132                 start += scope->length;
 133         }
 134         if (*cnt == 0)
 135                 return 0;
 136
 137         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 138         if (!*devices)
 139                 return -ENOMEM;
 140
 141         start = tmp;
 142         index = 0;
 143         while (start < end) {
 144                 scope = start;
 145                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 146                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 147                         ret = dmar_parse_one_dev_scope(scope,
 148                                 &(*devices)[index], segment);
 149                         if (ret) {
 150                                 kfree(*devices);
 151                                 return ret;
 152                         }
 153                         index ++;
 154                 }
 155                 start += scope->length;
 156         }
 157
 158         return 0;
 159 }
 160
 161 /**
 162  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 163  * structure which uniquely represent one DMA remapping hardware unit
 164  * present in the platform
 165  */
 166 static int __init
 167 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 168 {
 169         struct acpi_dmar_hardware_unit *drhd;
 170         struct dmar_drhd_unit *dmaru;
 171         int ret = 0;
 172
 173         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 174         if (!dmaru)
 175                 return -ENOMEM;
 176
 177         dmaru->hdr = header;
 178         drhd = (struct acpi_dmar_hardware_unit *)header;
 179         dmaru->reg_base_addr = drhd->address;
 180         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 181
 182         ret = alloc_iommu(dmaru);
 183         if (ret) {
 184                 kfree(dmaru);
 185                 return ret;
 186         }
 187         dmar_register_drhd_unit(dmaru);
 188         return 0;
 189 }
 190
 191 static int __init
 192 dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 193 {
 194         struct acpi_dmar_hardware_unit *drhd;
 195         static int include_all;
 196         int ret = 0;
 197
 198         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 199
 200         if (!dmaru->include_all)
 201                 ret = dmar_parse_dev_scope((void *)(drhd + 1),
 202                                 ((void *)drhd) + drhd->header.length,
 203                                 &dmaru->devices_cnt, &dmaru->devices,
 204                                 drhd->segment);
 205         else {
 206                 /* Only allow one INCLUDE_ALL */
 207                 if (include_all) {
 208                         printk(KERN_WARNING PREFIX "Only one INCLUDE_ALL "
 209                                 "device scope is allowed\n");
 210                         ret = -EINVAL;
 211                 }
 212                 include_all = 1;
 213         }
 214
 215         if (ret) {
 216                 list_del(&dmaru->list);
 217                 kfree(dmaru);
 218         }
 219         return ret;
 220 }
 221
 222 #ifdef CONFIG_DMAR
 223 LIST_HEAD(dmar_rmrr_units);
 224
 225 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 226 {
 227         list_add(&rmrr->list, &dmar_rmrr_units);
 228 }
 229
 230
 231 static int __init
 232 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 233 {
 234         struct acpi_dmar_reserved_memory *rmrr;
 235         struct dmar_rmrr_unit *rmrru;
 236
 237         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 238         if (!rmrru)
 239                 return -ENOMEM;
 240
 241         rmrru->hdr = header;
 242         rmrr = (struct acpi_dmar_reserved_memory *)header;
 243         rmrru->base_address = rmrr->base_address;
 244         rmrru->end_address = rmrr->end_address;
 245
 246         dmar_register_rmrr_unit(rmrru);
 247         return 0;
 248 }
 249
 250 static int __init
 251 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 252 {
 253         struct acpi_dmar_reserved_memory *rmrr;
 254         int ret;
 255
 256         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 257         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 258                 ((void *)rmrr) + rmrr->header.length,
 259                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 260
 261         if (ret || (rmrru->devices_cnt == 0)) {
 262                 list_del(&rmrru->list);
 263                 kfree(rmrru);
 264         }
 265         return ret;
 266 }
 267 #endif
 268
 269 static void __init
 270 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 271 {
 272         struct acpi_dmar_hardware_unit *drhd;
 273         struct acpi_dmar_reserved_memory *rmrr;
 274
 275         switch (header->type) {
 276         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 277                 drhd = (struct acpi_dmar_hardware_unit *)header;
 278                 printk (KERN_INFO PREFIX
 279                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
 280                         drhd->flags, drhd->address);
 281                 break;
 282         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 283                 rmrr = (struct acpi_dmar_reserved_memory *)header;
 284
 285                 printk (KERN_INFO PREFIX
 286                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
 287                         rmrr->base_address, rmrr->end_address);
 288                 break;
 289         }
 290 }
 291
 292 /**
 293  * dmar_table_detect - checks to see if the platform supports DMAR devices
 294  */
 295 static int __init dmar_table_detect(void)
 296 {
 297         acpi_status status = AE_OK;
 298
 299         /* if we could find DMAR table, then there are DMAR devices */
 300         status = acpi_get_table(ACPI_SIG_DMAR, 0,
 301                                 (struct acpi_table_header **)&dmar_tbl);
 302
 303         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 304                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 305                 status = AE_NOT_FOUND;
 306         }
 307
 308         return (ACPI_SUCCESS(status) ? 1 : 0);
 309 }
 310
 311 /**
 312  * parse_dmar_table - parses the DMA reporting table
 313  */
 314 static int __init
 315 parse_dmar_table(void)
 316 {
 317         struct acpi_table_dmar *dmar;
 318         struct acpi_dmar_header *entry_header;
 319         int ret = 0;
 320
 321         /*
 322          * Do it again, earlier dmar_tbl mapping could be mapped with
 323          * fixed map.
 324          */
 325         dmar_table_detect();
 326
 327         dmar = (struct acpi_table_dmar *)dmar_tbl;
 328         if (!dmar)
 329                 return -ENODEV;
 330
 331         if (dmar->width < PAGE_SHIFT_4K - 1) {
 332                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 333                 return -EINVAL;
 334         }
 335
 336         printk (KERN_INFO PREFIX "Host address width %d\n",
 337                 dmar->width + 1);
 338
 339         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 340         while (((unsigned long)entry_header) <
 341                         (((unsigned long)dmar) + dmar_tbl->length)) {
 342                 dmar_table_print_dmar_entry(entry_header);
 343
 344                 switch (entry_header->type) {
 345                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 346                         ret = dmar_parse_one_drhd(entry_header);
 347                         break;
 348                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 349 #ifdef CONFIG_DMAR
 350                         ret = dmar_parse_one_rmrr(entry_header);
 351 #endif
 352                         break;
 353                 default:
 354                         printk(KERN_WARNING PREFIX
 355                                 "Unknown DMAR structure type\n");
 356                         ret = 0; /* for forward compatibility */
 357                         break;
 358                 }
 359                 if (ret)
 360                         break;
 361
 362                 entry_header = ((void *)entry_header + entry_header->length);
 363         }
 364         return ret;
 365 }
 366
 367 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 368                           struct pci_dev *dev)
 369 {
 370         int index;
 371
 372         while (dev) {
 373                 for (index = 0; index < cnt; index++)
 374                         if (dev == devices[index])
 375                                 return 1;
 376
 377                 /* Check our parent */
 378                 dev = dev->bus->self;
 379         }
 380
 381         return 0;
 382 }
 383
 384 struct dmar_drhd_unit *
 385 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 386 {
 387         struct dmar_drhd_unit *drhd = NULL;
 388
 389         list_for_each_entry(drhd, &dmar_drhd_units, list) {
 390                 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
 391                                                 drhd->devices_cnt, dev))
 392                         return drhd;
 393         }
 394
 395         return NULL;
 396 }
 397
 398 int __init dmar_dev_scope_init(void)
 399 {
 400         struct dmar_drhd_unit *drhd, *drhd_n;
 401         int ret = -ENODEV;
 402
 403         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 404                 ret = dmar_parse_dev(drhd);
 405                 if (ret)
 406                         return ret;
 407         }
 408
 409 #ifdef CONFIG_DMAR
 410         {
 411                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 412                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 413                         ret = rmrr_parse_dev(rmrr);
 414                         if (ret)
 415                                 return ret;
 416                 }
 417         }
 418 #endif
 419
 420         return ret;
 421 }
 422
 423
 424 int __init dmar_table_init(void)
 425 {
 426         static int dmar_table_initialized;
 427         int ret;
 428
 429         if (dmar_table_initialized)
 430                 return 0;
 431
 432         dmar_table_initialized = 1;
 433
 434         ret = parse_dmar_table();
 435         if (ret) {
 436                 if (ret != -ENODEV)
 437                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 438                 return ret;
 439         }
 440
 441         if (list_empty(&dmar_drhd_units)) {
 442                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 443                 return -ENODEV;
 444         }
 445
 446 #ifdef CONFIG_DMAR
 447         if (list_empty(&dmar_rmrr_units))
 448                 printk(KERN_INFO PREFIX "No RMRR found\n");
 449 #endif
 450
 451 #ifdef CONFIG_INTR_REMAP
 452         parse_ioapics_under_ir();
 453 #endif
 454         return 0;
 455 }
 456
 457 void __init detect_intel_iommu(void)
 458 {
 459         int ret;
 460
 461         ret = dmar_table_detect();
 462
 463 #ifdef CONFIG_DMAR
 464         {
 465                 struct acpi_table_dmar *dmar;
 466                 /*
 467                  * for now we will disable dma-remapping when interrupt
 468                  * remapping is enabled.
 469                  * When support for queued invalidation for IOTLB invalidation
 470                  * is added, we will not need this any more.
 471                  */
 472                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 473                 if (ret && cpu_has_x2apic && dmar->flags & 0x1) {
 474                         printk(KERN_INFO
 475                                "Queued invalidation will be enabled to support "
 476                                "x2apic and Intr-remapping.\n");
 477                         printk(KERN_INFO
 478                                "Disabling IOMMU detection, because of missing "
 479                                "queued invalidation support for IOTLB "
 480                                "invalidation\n");
 481                         printk(KERN_INFO
 482                                "Use \"nox2apic\", if you want to use Intel "
 483                                " IOMMU for DMA-remapping and don't care about "
 484                                " x2apic support\n");
 485
 486                         dmar_disabled = 1;
 487                         goto end;
 488                 }
 489
 490                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 491                     !dmar_disabled)
 492                         iommu_detected = 1;
 493         }
 494 end:
 495 #endif
 496         dmar_tbl = NULL;
 497 }
 498
 499
 500 int alloc_iommu(struct dmar_drhd_unit *drhd)
 501 {
 502         struct intel_iommu *iommu;
 503         int map_size;
 504         u32 ver;
 505         static int iommu_allocated = 0;
 506
 507         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 508         if (!iommu)
 509                 return -ENOMEM;
 510
 511         iommu->seq_id = iommu_allocated++;
 512
 513         iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
 514         if (!iommu->reg) {
 515                 printk(KERN_ERR "IOMMU: can't map the region\n");
 516                 goto error;
 517         }
 518         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 519         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 520
 521         /* the registers might be more than one page */
 522         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 523                 cap_max_fault_reg_offset(iommu->cap));
 524         map_size = PAGE_ALIGN_4K(map_size);
 525         if (map_size > PAGE_SIZE_4K) {
 526                 iounmap(iommu->reg);
 527                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 528                 if (!iommu->reg) {
 529                         printk(KERN_ERR "IOMMU: can't map the region\n");
 530                         goto error;
 531                 }
 532         }
 533
 534         ver = readl(iommu->reg + DMAR_VER_REG);
 535         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 536                 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 537                 iommu->cap, iommu->ecap);
 538
 539         spin_lock_init(&iommu->register_lock);
 540
 541         drhd->iommu = iommu;
 542         return 0;
 543 error:
 544         kfree(iommu);
 545         return -1;
 546 }
 547
 548 void free_iommu(struct intel_iommu *iommu)
 549 {
 550         if (!iommu)
 551                 return;
 552
 553 #ifdef CONFIG_DMAR
 554         free_dmar_iommu(iommu);
 555 #endif
 556
 557         if (iommu->reg)
 558                 iounmap(iommu->reg);
 559         kfree(iommu);
 560 }
 561
 562 /*
 563  * Reclaim all the submitted descriptors which have completed its work.
 564  */
 565 static inline void reclaim_free_desc(struct q_inval *qi)
 566 {
 567         while (qi->desc_status[qi->free_tail] == QI_DONE) {
 568                 qi->desc_status[qi->free_tail] = QI_FREE;
 569                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 570                 qi->free_cnt++;
 571         }
 572 }
 573
 574 /*
 575  * Submit the queued invalidation descriptor to the remapping
 576  * hardware unit and wait for its completion.
 577  */
 578 void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 579 {
 580         struct q_inval *qi = iommu->qi;
 581         struct qi_desc *hw, wait_desc;
 582         int wait_index, index;
 583         unsigned long flags;
 584
 585         if (!qi)
 586                 return;
 587
 588         hw = qi->desc;
 589
 590         spin_lock(&qi->q_lock);
 591         while (qi->free_cnt < 3) {
 592                 spin_unlock(&qi->q_lock);
 593                 cpu_relax();
 594                 spin_lock(&qi->q_lock);
 595         }
 596
 597         index = qi->free_head;
 598         wait_index = (index + 1) % QI_LENGTH;
 599
 600         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 601
 602         hw[index] = *desc;
 603
 604         wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 605         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 606
 607         hw[wait_index] = wait_desc;
 608
 609         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 610         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 611
 612         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 613         qi->free_cnt -= 2;
 614
 615         spin_lock_irqsave(&iommu->register_lock, flags);
 616         /*
 617          * update the HW tail register indicating the presence of
 618          * new descriptors.
 619          */
 620         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
 621         spin_unlock_irqrestore(&iommu->register_lock, flags);
 622
 623         while (qi->desc_status[wait_index] != QI_DONE) {
 624                 spin_unlock(&qi->q_lock);
 625                 cpu_relax();
 626                 spin_lock(&qi->q_lock);
 627         }
 628
 629         qi->desc_status[index] = QI_DONE;
 630
 631         reclaim_free_desc(qi);
 632         spin_unlock(&qi->q_lock);
 633 }
 634
 635 /*
 636  * Flush the global interrupt entry cache.
 637  */
 638 void qi_global_iec(struct intel_iommu *iommu)
 639 {
 640         struct qi_desc desc;
 641
 642         desc.low = QI_IEC_TYPE;
 643         desc.high = 0;
 644
 645         qi_submit_sync(&desc, iommu);
 646 }
 647
 648 /*
 649  * Enable Queued Invalidation interface. This is a must to support
 650  * interrupt-remapping. Also used by DMA-remapping, which replaces
 651  * register based IOTLB invalidation.
 652  */
 653 int dmar_enable_qi(struct intel_iommu *iommu)
 654 {
 655         u32 cmd, sts;
 656         unsigned long flags;
 657         struct q_inval *qi;
 658
 659         if (!ecap_qis(iommu->ecap))
 660                 return -ENOENT;
 661
 662         /*
 663          * queued invalidation is already setup and enabled.
 664          */
 665         if (iommu->qi)
 666                 return 0;
 667
 668         iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
 669         if (!iommu->qi)
 670                 return -ENOMEM;
 671
 672         qi = iommu->qi;
 673
 674         qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
 675         if (!qi->desc) {
 676                 kfree(qi);
 677                 iommu->qi = 0;
 678                 return -ENOMEM;
 679         }
 680
 681         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
 682         if (!qi->desc_status) {
 683                 free_page((unsigned long) qi->desc);
 684                 kfree(qi);
 685                 iommu->qi = 0;
 686                 return -ENOMEM;
 687         }
 688
 689         qi->free_head = qi->free_tail = 0;
 690         qi->free_cnt = QI_LENGTH;
 691
 692         spin_lock_init(&qi->q_lock);
 693
 694         spin_lock_irqsave(&iommu->register_lock, flags);
 695         /* write zero to the tail reg */
 696         writel(0, iommu->reg + DMAR_IQT_REG);
 697
 698         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 699
 700         cmd = iommu->gcmd | DMA_GCMD_QIE;
 701         iommu->gcmd |= DMA_GCMD_QIE;
 702         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 703
 704         /* Make sure hardware complete it */
 705         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 706         spin_unlock_irqrestore(&iommu->register_lock, flags);
 707
 708         return 0;
 709 }