1 #include <linux/dma-mapping.h>
2 #include <linux/dmar.h>
3 #include <linux/bootmem.h>
9 #include <asm/calgary.h>
10 #include <asm/amd_iommu.h>
12 static int forbid_dac __read_mostly;
14 struct dma_mapping_ops *dma_ops;
15 EXPORT_SYMBOL(dma_ops);
17 static int iommu_sac_force __read_mostly;
19 #ifdef CONFIG_IOMMU_DEBUG
20 int panic_on_overflow __read_mostly = 1;
21 int force_iommu __read_mostly = 1;
23 int panic_on_overflow __read_mostly = 0;
24 int force_iommu __read_mostly = 0;
27 int iommu_merge __read_mostly = 0;
29 int no_iommu __read_mostly;
30 /* Set this to 1 if there is a HW IOMMU in the system */
31 int iommu_detected __read_mostly = 0;
33 /* This tells the BIO block layer to assume merging. Default to off
34 because we cannot guarantee merging later. */
35 int iommu_bio_merge __read_mostly = 0;
36 EXPORT_SYMBOL(iommu_bio_merge);
38 dma_addr_t bad_dma_address __read_mostly = 0;
39 EXPORT_SYMBOL(bad_dma_address);
41 /* Dummy device used for NULL arguments (normally ISA). Better would
42 be probably a smaller DMA mask, but this is bug-to-bug compatible
44 struct device fallback_dev = {
45 .bus_id = "fallback device",
46 .coherent_dma_mask = DMA_32BIT_MASK,
47 .dma_mask = &fallback_dev.coherent_dma_mask,
50 int dma_set_mask(struct device *dev, u64 mask)
52 if (!dev->dma_mask || !dma_supported(dev, mask))
55 *dev->dma_mask = mask;
59 EXPORT_SYMBOL(dma_set_mask);
62 static __initdata void *dma32_bootmem_ptr;
63 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
65 static int __init parse_dma32_size_opt(char *p)
69 dma32_bootmem_size = memparse(p, &p);
72 early_param("dma32_size", parse_dma32_size_opt);
74 void __init dma32_reserve_bootmem(void)
76 unsigned long size, align;
77 if (max_pfn <= MAX_DMA32_PFN)
81 * check aperture_64.c allocate_aperture() for reason about
85 size = round_up(dma32_bootmem_size, align);
86 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
88 if (dma32_bootmem_ptr)
89 dma32_bootmem_size = size;
91 dma32_bootmem_size = 0;
93 static void __init dma32_free_bootmem(void)
96 if (max_pfn <= MAX_DMA32_PFN)
99 if (!dma32_bootmem_ptr)
102 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
104 dma32_bootmem_ptr = NULL;
105 dma32_bootmem_size = 0;
108 void __init pci_iommu_alloc(void)
110 /* free the range so iommu could get some range less than 4G */
111 dma32_free_bootmem();
113 * The order of these functions is important for
114 * fall-back/fail-over reasons
116 gart_iommu_hole_init();
120 detect_intel_iommu();
127 unsigned long iommu_num_pages(unsigned long addr, unsigned long len)
129 unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
131 return size >> PAGE_SHIFT;
133 EXPORT_SYMBOL(iommu_num_pages);
137 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
140 static __init int iommu_setup(char *p)
148 if (!strncmp(p, "off", 3))
150 /* gart_parse_options has more force support */
151 if (!strncmp(p, "force", 5))
153 if (!strncmp(p, "noforce", 7)) {
158 if (!strncmp(p, "biomerge", 8)) {
159 iommu_bio_merge = 4096;
163 if (!strncmp(p, "panic", 5))
164 panic_on_overflow = 1;
165 if (!strncmp(p, "nopanic", 7))
166 panic_on_overflow = 0;
167 if (!strncmp(p, "merge", 5)) {
171 if (!strncmp(p, "nomerge", 7))
173 if (!strncmp(p, "forcesac", 8))
175 if (!strncmp(p, "allowdac", 8))
177 if (!strncmp(p, "nodac", 5))
179 if (!strncmp(p, "usedac", 6)) {
183 #ifdef CONFIG_SWIOTLB
184 if (!strncmp(p, "soft", 4))
188 gart_parse_options(p);
190 #ifdef CONFIG_CALGARY_IOMMU
191 if (!strncmp(p, "calgary", 7))
193 #endif /* CONFIG_CALGARY_IOMMU */
195 p += strcspn(p, ",");
201 early_param("iommu", iommu_setup);
203 int dma_supported(struct device *dev, u64 mask)
205 struct dma_mapping_ops *ops = get_dma_ops(dev);
208 if (mask > 0xffffffff && forbid_dac > 0) {
209 dev_info(dev, "PCI: Disallowing DAC for device\n");
214 if (ops->dma_supported)
215 return ops->dma_supported(dev, mask);
217 /* Copied from i386. Doesn't make much sense, because it will
218 only work for pci_alloc_coherent.
219 The caller just has to use GFP_DMA in this case. */
220 if (mask < DMA_24BIT_MASK)
223 /* Tell the device to use SAC when IOMMU force is on. This
224 allows the driver to use cheaper accesses in some cases.
226 Problem with this is that if we overflow the IOMMU area and
227 return DAC as fallback address the device may not handle it
230 As a special case some controllers have a 39bit address
231 mode that is as efficient as 32bit (aic79xx). Don't force
232 SAC for these. Assume all masks <= 40 bits are of this
233 type. Normally this doesn't make any difference, but gives
234 more gentle handling of IOMMU overflow. */
235 if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
236 dev_info(dev, "Force SAC with mask %Lx\n", mask);
242 EXPORT_SYMBOL(dma_supported);
245 * Allocate memory for a coherent mapping.
248 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
251 struct dma_mapping_ops *ops = get_dma_ops(dev);
254 if (dma_alloc_from_coherent(dev, size, dma_handle, &memory))
262 if (ops->alloc_coherent)
263 return ops->alloc_coherent(dev, size,
267 EXPORT_SYMBOL(dma_alloc_coherent);
270 * Unmap coherent memory.
271 * The caller must ensure that the device has finished accessing the mapping.
273 void dma_free_coherent(struct device *dev, size_t size,
274 void *vaddr, dma_addr_t bus)
276 struct dma_mapping_ops *ops = get_dma_ops(dev);
278 WARN_ON(irqs_disabled()); /* for portability */
280 if (dma_release_from_coherent(dev, get_order(size), vaddr))
283 if (ops->free_coherent)
284 ops->free_coherent(dev, size, vaddr, bus);
286 EXPORT_SYMBOL(dma_free_coherent);
288 static int __init pci_iommu_init(void)
290 calgary_iommu_init();
302 void pci_iommu_shutdown(void)
304 gart_iommu_shutdown();
306 /* Must execute after PCI subsystem */
307 fs_initcall(pci_iommu_init);
310 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
312 static __devinit void via_no_dac(struct pci_dev *dev)
314 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
315 printk(KERN_INFO "PCI: VIA PCI bridge detected."
320 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);