Currently the calgary code can give drivers addresses above 4GB which is
very bad for hardware that is only 32bit DMA addressable.  This patch
"teaches" calgary to fallback to the appropriate dma_ops when it
encounters a device/bus which is not behind the Calgary/CalIOC2.  I
believe there is a better way to do this and am open for ideas, but for
now this certainly fixes the badness.


Signed-off-by Alexis Bruemmer <alexisb@us.ibm.com>

---

Index: linux-2.6.26-rc1/arch/x86/kernel/pci-nommu.c
===================================================================
--- linux-2.6.26-rc1.orig/arch/x86/kernel/pci-nommu.c
+++ linux-2.6.26-rc1/arch/x86/kernel/pci-nommu.c
@@ -10,6 +10,7 @@
 #include <asm/gart.h>
 #include <asm/processor.h>
 #include <asm/dma.h>
+#include <asm/calgary.h>

 static int
 check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size)
@@ -92,6 +93,10 @@ const struct dma_mapping_ops nommu_dma_o

 void __init no_iommu_init(void)
 {
+#ifdef CONFIG_CALGARY_IOMMU
+	if (use_calgary && (end_pfn < MAX_DMA32_PFN))
+		fallback_dma_ops = &nommu_dma_ops;
+#endif
 	if (dma_ops)
 		return;

Index: linux-2.6.26-rc1/arch/x86/kernel/pci-swiotlb_64.c
===================================================================
--- linux-2.6.26-rc1.orig/arch/x86/kernel/pci-swiotlb_64.c
+++ linux-2.6.26-rc1/arch/x86/kernel/pci-swiotlb_64.c
@@ -8,6 +8,7 @@
 #include <asm/gart.h>
 #include <asm/swiotlb.h>
 #include <asm/dma.h>
+#include <asm/calgary.h>

 int swiotlb __read_mostly;

@@ -47,4 +48,10 @@ void __init pci_swiotlb_init(void)
 		swiotlb_init();
 		dma_ops = &swiotlb_dma_ops;
 	}
+#ifdef CONFIG_CALGARY_IOMMU
+	else if (use_calgary && (end_pfn > MAX_DMA32_PFN)) {
+		swiotlb_init();
+		fallback_dma_ops = &swiotlb_dma_ops;
+	}
+#endif
 }
Index: linux-2.6.26-rc1/include/asm-x86/calgary.h
===================================================================
--- linux-2.6.2...
[ message continues ]

On Thu, 08 May 2008 14:40:20 -0700

I'm not sure that I correctly understand what you want. You mean that
the Calgary IOMMU code ignores device's dma_mask and gives addresses
above 4GB or the Calgary IOMMU code wrongly handles devices that are
not behind the Calgary?

If you refers to the former problem, other IOMMU implementations do
something like this.


diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c
index e28ec49..2a977bc 100644
--- a/arch/x86/kernel/pci-calgary_64.c
+++ b/arch/x86/kernel/pci-calgary_64.c
@@ -268,21 +268,37 @@ static unsigned long iommu_range_alloc(struct device *dev,
 {
 	unsigned long flags;
 	unsigned long offset;
+	unsigned long limit;
 	unsigned long boundary_size;
+	unsigned long start;
+	unsigned long mask;
 
 	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
 			      PAGE_SIZE) >> PAGE_SHIFT;
 
 	BUG_ON(npages == 0);
 
+	if (dev->dma_mask)
+		mask = *dev->dma_mask >> PAGE_SHIFT;
+	else
+		mask = 0xfffffffful >> PAGE_SHIFT;
+
+	limit = tbl->it_size;
+	start = tbl->it_hint;
+
+	if (limit > mask) {
+		limit = mask + 1;
+		start = tbl->it_hint & mask;
+	}
+
 	spin_lock_irqsave(&tbl->it_lock, flags);
 
-	offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
+	offset = iommu_area_alloc(tbl->it_map, limit, start,
 				  npages, 0, boundary_size, 0);
 	if (offset == ~0UL) {
 		tbl->chip_ops->tce_cache_blast(tbl);
 
-		offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
+		offset = iommu_area_alloc(tbl->it_map, limit, 0,
 					  npages, 0, boundary_size, 0);
 		if (offset == ~0UL) {
 			printk(KERN_WARNING "Calgary: IOMMU full.\n");
--

The real issue is the latter-- the Calgary IOMMU code does not properly

--

On Thu, 08 May 2008 16:41:51 -0700

Thanks, now I see why you use swiotlb for such devices in the case of
end_pfn > MAX_DMA32_PFN and no_dma_ops works for them in the case of
of end_pfn < MAX_DMA32_PFN.

Can we put a pointer to dma_ops in struct device (archdata) like POWER
does? The way to setup and handle x86 IOMMUs seems to become hacky day
by day.
--

Per-device dma-ops will be useful for KVM (for pass-through device
support) and IB as well.

Cheers,
Muli
--