2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
26 * These routines are used by both DMA-remapping and Interrupt-remapping
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
36 #define PREFIX "DMAR:"
38 /* No locks are needed as DMA remapping hardware unit
39 * list is constructed at boot time and hotplug of
40 * these units are not supported by the architecture.
42 LIST_HEAD(dmar_drhd_units);
44 static struct acpi_table_header * __initdata dmar_tbl;
46 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
49 * add INCLUDE_ALL at the tail, so scan the list will find it at
52 if (drhd->include_all)
53 list_add_tail(&drhd->list, &dmar_drhd_units);
55 list_add(&drhd->list, &dmar_drhd_units);
58 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
59 struct pci_dev **dev, u16 segment)
62 struct pci_dev *pdev = NULL;
63 struct acpi_dmar_pci_path *path;
66 bus = pci_find_bus(segment, scope->bus);
67 path = (struct acpi_dmar_pci_path *)(scope + 1);
68 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
69 / sizeof(struct acpi_dmar_pci_path);
75 * Some BIOSes list non-exist devices in DMAR table, just
80 PREFIX "Device scope bus [%d] not found\n",
84 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
86 printk(KERN_WARNING PREFIX
87 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
88 segment, bus->number, path->dev, path->fn);
93 bus = pdev->subordinate;
96 printk(KERN_WARNING PREFIX
97 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
98 segment, scope->bus, path->dev, path->fn);
102 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
103 pdev->subordinate) || (scope->entry_type == \
104 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
106 printk(KERN_WARNING PREFIX
107 "Device scope type does not match for %s\n",
115 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
116 struct pci_dev ***devices, u16 segment)
118 struct acpi_dmar_device_scope *scope;
124 while (start < end) {
126 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
127 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
130 printk(KERN_WARNING PREFIX
131 "Unsupported device scope\n");
132 start += scope->length;
137 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
143 while (start < end) {
145 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
146 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
147 ret = dmar_parse_one_dev_scope(scope,
148 &(*devices)[index], segment);
155 start += scope->length;
162 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
163 * structure which uniquely represent one DMA remapping hardware unit
164 * present in the platform
167 dmar_parse_one_drhd(struct acpi_dmar_header *header)
169 struct acpi_dmar_hardware_unit *drhd;
170 struct dmar_drhd_unit *dmaru;
173 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
178 drhd = (struct acpi_dmar_hardware_unit *)header;
179 dmaru->reg_base_addr = drhd->address;
180 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
182 ret = alloc_iommu(dmaru);
187 dmar_register_drhd_unit(dmaru);
192 dmar_parse_dev(struct dmar_drhd_unit *dmaru)
194 struct acpi_dmar_hardware_unit *drhd;
195 static int include_all;
198 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
200 if (!dmaru->include_all)
201 ret = dmar_parse_dev_scope((void *)(drhd + 1),
202 ((void *)drhd) + drhd->header.length,
203 &dmaru->devices_cnt, &dmaru->devices,
206 /* Only allow one INCLUDE_ALL */
208 printk(KERN_WARNING PREFIX "Only one INCLUDE_ALL "
209 "device scope is allowed\n");
215 if (ret || (dmaru->devices_cnt == 0 && !dmaru->include_all)) {
216 list_del(&dmaru->list);
223 LIST_HEAD(dmar_rmrr_units);
225 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
227 list_add(&rmrr->list, &dmar_rmrr_units);
232 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
234 struct acpi_dmar_reserved_memory *rmrr;
235 struct dmar_rmrr_unit *rmrru;
237 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
242 rmrr = (struct acpi_dmar_reserved_memory *)header;
243 rmrru->base_address = rmrr->base_address;
244 rmrru->end_address = rmrr->end_address;
246 dmar_register_rmrr_unit(rmrru);
251 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
253 struct acpi_dmar_reserved_memory *rmrr;
256 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
257 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
258 ((void *)rmrr) + rmrr->header.length,
259 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
261 if (ret || (rmrru->devices_cnt == 0)) {
262 list_del(&rmrru->list);
270 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
272 struct acpi_dmar_hardware_unit *drhd;
273 struct acpi_dmar_reserved_memory *rmrr;
275 switch (header->type) {
276 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
277 drhd = (struct acpi_dmar_hardware_unit *)header;
278 printk (KERN_INFO PREFIX
279 "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
280 drhd->flags, (unsigned long long)drhd->address);
282 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
283 rmrr = (struct acpi_dmar_reserved_memory *)header;
285 printk (KERN_INFO PREFIX
286 "RMRR base: 0x%016Lx end: 0x%016Lx\n",
287 (unsigned long long)rmrr->base_address,
288 (unsigned long long)rmrr->end_address);
295 * parse_dmar_table - parses the DMA reporting table
298 parse_dmar_table(void)
300 struct acpi_table_dmar *dmar;
301 struct acpi_dmar_header *entry_header;
304 dmar = (struct acpi_table_dmar *)dmar_tbl;
308 if (dmar->width < PAGE_SHIFT - 1) {
309 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
313 printk (KERN_INFO PREFIX "Host address width %d\n",
316 entry_header = (struct acpi_dmar_header *)(dmar + 1);
317 while (((unsigned long)entry_header) <
318 (((unsigned long)dmar) + dmar_tbl->length)) {
319 dmar_table_print_dmar_entry(entry_header);
321 switch (entry_header->type) {
322 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
323 ret = dmar_parse_one_drhd(entry_header);
325 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
327 ret = dmar_parse_one_rmrr(entry_header);
331 printk(KERN_WARNING PREFIX
332 "Unknown DMAR structure type\n");
333 ret = 0; /* for forward compatibility */
339 entry_header = ((void *)entry_header + entry_header->length);
344 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
350 for (index = 0; index < cnt; index++)
351 if (dev == devices[index])
354 /* Check our parent */
355 dev = dev->bus->self;
361 struct dmar_drhd_unit *
362 dmar_find_matched_drhd_unit(struct pci_dev *dev)
364 struct dmar_drhd_unit *drhd = NULL;
366 list_for_each_entry(drhd, &dmar_drhd_units, list) {
367 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
368 drhd->devices_cnt, dev))
375 int __init dmar_dev_scope_init(void)
377 struct dmar_drhd_unit *drhd;
380 for_each_drhd_unit(drhd) {
381 ret = dmar_parse_dev(drhd);
388 struct dmar_rmrr_unit *rmrr;
389 for_each_rmrr_units(rmrr) {
390 ret = rmrr_parse_dev(rmrr);
401 int __init dmar_table_init(void)
403 static int dmar_table_initialized;
406 if (dmar_table_initialized)
409 dmar_table_initialized = 1;
411 ret = parse_dmar_table();
414 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
418 if (list_empty(&dmar_drhd_units)) {
419 printk(KERN_INFO PREFIX "No DMAR devices found\n");
424 if (list_empty(&dmar_rmrr_units))
425 printk(KERN_INFO PREFIX "No RMRR found\n");
428 #ifdef CONFIG_INTR_REMAP
429 parse_ioapics_under_ir();
435 * early_dmar_detect - checks to see if the platform supports DMAR devices
437 int __init early_dmar_detect(void)
439 acpi_status status = AE_OK;
441 /* if we could find DMAR table, then there are DMAR devices */
442 status = acpi_get_table(ACPI_SIG_DMAR, 0,
443 (struct acpi_table_header **)&dmar_tbl);
445 if (ACPI_SUCCESS(status) && !dmar_tbl) {
446 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
447 status = AE_NOT_FOUND;
450 return (ACPI_SUCCESS(status) ? 1 : 0);
453 void __init detect_intel_iommu(void)
457 ret = early_dmar_detect();
460 #ifdef CONFIG_INTR_REMAP
461 struct acpi_table_dmar *dmar;
463 * for now we will disable dma-remapping when interrupt
464 * remapping is enabled.
465 * When support for queued invalidation for IOTLB invalidation
466 * is added, we will not need this any more.
468 dmar = (struct acpi_table_dmar *) dmar_tbl;
469 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
471 "Queued invalidation will be enabled to support "
472 "x2apic and Intr-remapping.\n");
476 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
484 int alloc_iommu(struct dmar_drhd_unit *drhd)
486 struct intel_iommu *iommu;
489 static int iommu_allocated = 0;
491 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
495 iommu->seq_id = iommu_allocated++;
497 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
499 printk(KERN_ERR "IOMMU: can't map the region\n");
502 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
503 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
505 /* the registers might be more than one page */
506 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
507 cap_max_fault_reg_offset(iommu->cap));
508 map_size = VTD_PAGE_ALIGN(map_size);
509 if (map_size > VTD_PAGE_SIZE) {
511 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
513 printk(KERN_ERR "IOMMU: can't map the region\n");
518 ver = readl(iommu->reg + DMAR_VER_REG);
519 pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
520 (unsigned long long)drhd->reg_base_addr,
521 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
522 (unsigned long long)iommu->cap,
523 (unsigned long long)iommu->ecap);
525 spin_lock_init(&iommu->register_lock);
534 void free_iommu(struct intel_iommu *iommu)
540 free_dmar_iommu(iommu);
549 * Reclaim all the submitted descriptors which have completed its work.
551 static inline void reclaim_free_desc(struct q_inval *qi)
553 while (qi->desc_status[qi->free_tail] == QI_DONE) {
554 qi->desc_status[qi->free_tail] = QI_FREE;
555 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
561 * Submit the queued invalidation descriptor to the remapping
562 * hardware unit and wait for its completion.
564 void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
566 struct q_inval *qi = iommu->qi;
567 struct qi_desc *hw, wait_desc;
568 int wait_index, index;
576 spin_lock_irqsave(&qi->q_lock, flags);
577 while (qi->free_cnt < 3) {
578 spin_unlock_irqrestore(&qi->q_lock, flags);
580 spin_lock_irqsave(&qi->q_lock, flags);
583 index = qi->free_head;
584 wait_index = (index + 1) % QI_LENGTH;
586 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
590 wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
591 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
593 hw[wait_index] = wait_desc;
595 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
596 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
598 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
601 spin_lock(&iommu->register_lock);
603 * update the HW tail register indicating the presence of
606 writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
607 spin_unlock(&iommu->register_lock);
609 while (qi->desc_status[wait_index] != QI_DONE) {
611 * We will leave the interrupts disabled, to prevent interrupt
612 * context to queue another cmd while a cmd is already submitted
613 * and waiting for completion on this cpu. This is to avoid
614 * a deadlock where the interrupt context can wait indefinitely
615 * for free slots in the queue.
617 spin_unlock(&qi->q_lock);
619 spin_lock(&qi->q_lock);
622 qi->desc_status[index] = QI_DONE;
624 reclaim_free_desc(qi);
625 spin_unlock_irqrestore(&qi->q_lock, flags);
629 * Flush the global interrupt entry cache.
631 void qi_global_iec(struct intel_iommu *iommu)
635 desc.low = QI_IEC_TYPE;
638 qi_submit_sync(&desc, iommu);
641 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
642 u64 type, int non_present_entry_flush)
647 if (non_present_entry_flush) {
648 if (!cap_caching_mode(iommu->cap))
654 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
655 | QI_CC_GRAN(type) | QI_CC_TYPE;
658 qi_submit_sync(&desc, iommu);
664 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
665 unsigned int size_order, u64 type,
666 int non_present_entry_flush)
673 if (non_present_entry_flush) {
674 if (!cap_caching_mode(iommu->cap))
680 if (cap_write_drain(iommu->cap))
683 if (cap_read_drain(iommu->cap))
686 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
687 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
688 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
689 | QI_IOTLB_AM(size_order);
691 qi_submit_sync(&desc, iommu);
698 * Enable Queued Invalidation interface. This is a must to support
699 * interrupt-remapping. Also used by DMA-remapping, which replaces
700 * register based IOTLB invalidation.
702 int dmar_enable_qi(struct intel_iommu *iommu)
708 if (!ecap_qis(iommu->ecap))
712 * queued invalidation is already setup and enabled.
717 iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
723 qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
730 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
731 if (!qi->desc_status) {
732 free_page((unsigned long) qi->desc);
738 qi->free_head = qi->free_tail = 0;
739 qi->free_cnt = QI_LENGTH;
741 spin_lock_init(&qi->q_lock);
743 spin_lock_irqsave(&iommu->register_lock, flags);
744 /* write zero to the tail reg */
745 writel(0, iommu->reg + DMAR_IQT_REG);
747 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
749 cmd = iommu->gcmd | DMA_GCMD_QIE;
750 iommu->gcmd |= DMA_GCMD_QIE;
751 writel(cmd, iommu->reg + DMAR_GCMD_REG);
753 /* Make sure hardware complete it */
754 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
755 spin_unlock_irqrestore(&iommu->register_lock, flags);