mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-09 21:59:12 -04:00
iommu/io-pgtable-fast: optimize statically allocated pages
Presently fastmap iommu feature allocates page tables for full 4GB virtual address space. This can be optimized to consider virtual address range [base, size] needed by client and prepare page tables only for applicable region. CRs-Fixed: 2011508 Change-Id: Ie6c23cb8e1702a823567e126f452b1e72d851f71 Signed-off-by: Shiraz Hashim <shashim@codeaurora.org> Signed-off-by: Prakash Gupta <guptap@codeaurora.org> [@lmark@codeaurora.org: modify downstream struct instead] Signed-off-by: Liam Mark <lmark@codeaurora.org>
This commit is contained in:
Shiraz Hashim
Liam Mark
parent
bd0cb90137
commit
7416fc8014
5 changed files with 217 additions and 90 deletions
|
|
@ -1740,72 +1740,140 @@ static void arm_smmu_free_asid(struct iommu_domain *domain)
|
|||
mutex_unlock(&smmu->idr_mutex);
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks for "qcom,iommu-dma-addr-pool" property to specify the IOVA range
|
||||
* for the domain. If not present, and the domain doesn't use fastmap,
|
||||
* the domain geometry is unmodified.
|
||||
*/
|
||||
static int arm_smmu_adjust_domain_geometry(struct device *dev,
|
||||
struct iommu_domain *domain)
|
||||
static int get_range_prop(struct device *dev, const char *prop,
|
||||
dma_addr_t *ret_base, dma_addr_t *ret_end)
|
||||
{
|
||||
struct device_node *np;
|
||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
||||
int naddr, nsize, len;
|
||||
u64 dma_base, dma_size, dma_end;
|
||||
u64 base, end, size;
|
||||
const __be32 *ranges;
|
||||
dma_addr_t hw_base = domain->geometry.aperture_start;
|
||||
dma_addr_t hw_end = domain->geometry.aperture_end;
|
||||
bool is_fast = test_bit(DOMAIN_ATTR_FAST, smmu_domain->attributes);
|
||||
|
||||
if (!dev->of_node)
|
||||
return 0;
|
||||
return -ENOENT;
|
||||
|
||||
np = of_parse_phandle(dev->of_node, "qcom,iommu-group", 0);
|
||||
if (!np)
|
||||
np = dev->of_node;
|
||||
|
||||
ranges = of_get_property(np, "qcom,iommu-dma-addr-pool", &len);
|
||||
ranges = of_get_property(np, prop, &len);
|
||||
|
||||
if (!ranges && !is_fast)
|
||||
return 0;
|
||||
if (!ranges)
|
||||
return -ENOENT;
|
||||
|
||||
if (ranges) {
|
||||
len /= sizeof(u32);
|
||||
naddr = of_n_addr_cells(np);
|
||||
nsize = of_n_size_cells(np);
|
||||
if (len < naddr + nsize) {
|
||||
dev_err(dev, "Invalid length for qcom,iommu-dma-addr-pool, expected %d cells\n",
|
||||
naddr + nsize);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (naddr == 0 || nsize == 0) {
|
||||
dev_err(dev, "Invalid #address-cells %d or #size-cells %d\n",
|
||||
naddr, nsize);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
dma_base = of_read_number(ranges, naddr);
|
||||
dma_size = of_read_number(ranges + naddr, nsize);
|
||||
dma_end = dma_base + dma_size - 1;
|
||||
} else {
|
||||
/*
|
||||
* This domain uses fastmap, but doesn't have any domain
|
||||
* geometry limitations, as implied by the absence of the
|
||||
* qcom,iommu-dma-addr-pool property, so impose the default
|
||||
* fastmap geometry requirement.
|
||||
*/
|
||||
dma_base = 0;
|
||||
dma_end = SZ_4G - 1;
|
||||
len /= sizeof(u32);
|
||||
naddr = of_n_addr_cells(np);
|
||||
nsize = of_n_size_cells(np);
|
||||
if (len < naddr + nsize) {
|
||||
dev_err(dev, "Invalid length for %s, expected %d cells\n",
|
||||
prop, naddr + nsize);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (naddr == 0 || nsize == 0) {
|
||||
dev_err(dev, "Invalid #address-cells %d or #size-cells %d for %s\n",
|
||||
prop, naddr, nsize);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
base = of_read_number(ranges, naddr);
|
||||
size = of_read_number(ranges + naddr, nsize);
|
||||
end = base + size - 1;
|
||||
|
||||
*ret_base = base;
|
||||
*ret_end = end;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int arm_smmu_get_domain_dma_range(struct device *dev,
|
||||
struct iommu_domain *domain,
|
||||
dma_addr_t hw_base,
|
||||
dma_addr_t hw_end,
|
||||
dma_addr_t *ret_base,
|
||||
dma_addr_t *ret_end)
|
||||
{
|
||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
||||
dma_addr_t dma_base, dma_end;
|
||||
bool is_fast = test_bit(DOMAIN_ATTR_FAST, smmu_domain->attributes);
|
||||
int ret;
|
||||
|
||||
ret = get_range_prop(dev, "qcom,iommu-dma-addr-pool", &dma_base,
|
||||
&dma_end);
|
||||
if (ret == -ENOENT) {
|
||||
if (is_fast) {
|
||||
/*
|
||||
* This domain uses fastmap, but doesn't have any domain
|
||||
* geometry limitations, as implied by the absence of
|
||||
* the qcom,iommu-dma-addr-pool property, so impose the
|
||||
* default fastmap geometry requirement.
|
||||
*/
|
||||
dma_base = 0;
|
||||
dma_end = SZ_4G - 1;
|
||||
} else {
|
||||
dma_base = hw_base;
|
||||
dma_end = hw_end;
|
||||
}
|
||||
} else if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
/*
|
||||
* The original geometry describes the IOVA limitations of the hardware,
|
||||
* so lets make sure that the IOVA range for this device is at least
|
||||
* within those bounds.
|
||||
*/
|
||||
if (!((hw_base <= dma_base) && (dma_end <= hw_end)))
|
||||
return -EINVAL;
|
||||
|
||||
*ret_base = dma_base;
|
||||
*ret_end = dma_end;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the supported IOVA range for the domain, this can be larger than the
|
||||
* configured DMA layer IOVA range.
|
||||
*/
|
||||
static int arm_smmu_get_domain_iova_range(struct device *dev,
|
||||
struct iommu_domain *domain,
|
||||
unsigned long ias,
|
||||
dma_addr_t *ret_base,
|
||||
dma_addr_t *ret_end)
|
||||
{
|
||||
dma_addr_t iova_base, iova_end;
|
||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
||||
dma_addr_t hw_base = 0;
|
||||
dma_addr_t hw_end = (1UL << ias) - 1;
|
||||
bool is_fast = test_bit(DOMAIN_ATTR_FAST, smmu_domain->attributes);
|
||||
|
||||
if (is_fast) {
|
||||
iova_base = 0;
|
||||
iova_end = SZ_4G - 1;
|
||||
} else {
|
||||
iova_base = hw_base;
|
||||
iova_end = hw_end;
|
||||
}
|
||||
|
||||
if (!((hw_base <= iova_base) && (iova_end <= hw_end)))
|
||||
return -EINVAL;
|
||||
|
||||
*ret_base = iova_base;
|
||||
*ret_end = iova_end;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Checks for "qcom,iommu-dma-addr-pool" property to specify the DMA layer IOVA
|
||||
* range for the domain. If not present, and the domain doesn't use fastmap,
|
||||
* the domain geometry is unmodified.
|
||||
*/
|
||||
static int arm_smmu_adjust_domain_geometry(struct device *dev,
|
||||
struct iommu_domain *domain)
|
||||
{
|
||||
dma_addr_t dma_base, dma_end;
|
||||
int ret;
|
||||
|
||||
ret = arm_smmu_get_domain_dma_range(dev, domain,
|
||||
domain->geometry.aperture_start,
|
||||
domain->geometry.aperture_end,
|
||||
&dma_base, &dma_end);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
domain->geometry.aperture_start = dma_base;
|
||||
domain->geometry.aperture_end = dma_end;
|
||||
return 0;
|
||||
|
|
@ -2060,6 +2128,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
|
|||
goto out_clear_smmu;
|
||||
}
|
||||
|
||||
ret = arm_smmu_get_domain_iova_range(dev, domain, ias,
|
||||
&ttbr0_pgtbl_info->iova_base,
|
||||
&ttbr0_pgtbl_info->iova_end);
|
||||
if (ret)
|
||||
goto out_clear_smmu;
|
||||
|
||||
ttbr0_pgtbl_info->pgtbl_cfg = (struct io_pgtable_cfg) {
|
||||
.quirks = quirks,
|
||||
.pgsize_bitmap = smmu->pgsize_bitmap,
|
||||
|
|
@ -2079,6 +2153,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
|
|||
goto out_clear_smmu;
|
||||
}
|
||||
if (split_tables) {
|
||||
ttbr1_pgtbl_info->iova_base = ttbr0_pgtbl_info->iova_base;
|
||||
ttbr1_pgtbl_info->iova_end = ttbr0_pgtbl_info->iova_end;
|
||||
ttbr1_pgtbl_info->pgtbl_cfg = ttbr0_pgtbl_info->pgtbl_cfg;
|
||||
smmu_domain->pgtbl_ops[1] = alloc_io_pgtable_ops(fmt,
|
||||
&ttbr1_pgtbl_info->pgtbl_cfg,
|
||||
|
|
|
|||
|
|
@ -116,6 +116,9 @@ static dma_addr_t __fast_smmu_alloc_iova(struct dma_fast_smmu_mapping *mapping,
|
|||
mapping->num_4k_pages);
|
||||
mapping->have_stale_tlbs = false;
|
||||
av8l_fast_clear_stale_ptes(mapping->pgtbl_ops,
|
||||
mapping->base,
|
||||
mapping->base +
|
||||
mapping->size - 1,
|
||||
skip_sync);
|
||||
bit = bitmap_find_next_zero_area(
|
||||
mapping->clean_bitmap,
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
#include <linux/types.h>
|
||||
#include <linux/io-pgtable.h>
|
||||
#include <linux/io-pgtable-fast.h>
|
||||
#include <linux/mm.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/dma-mapping.h>
|
||||
|
|
@ -126,7 +127,7 @@
|
|||
|
||||
#define PTE_SH_IDX(pte) (pte & AV8L_FAST_PTE_SH_MASK)
|
||||
|
||||
#define iopte_pmd_offset(pmds, iova) (pmds + (iova >> 12))
|
||||
#define iopte_pmd_offset(pmds, base, iova) (pmds + ((iova - base) >> 12))
|
||||
|
||||
static inline dma_addr_t av8l_dma_addr(void *addr)
|
||||
{
|
||||
|
|
@ -193,14 +194,16 @@ static void __av8l_check_for_stale_tlb(av8l_fast_iopte *ptep)
|
|||
}
|
||||
}
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync)
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base,
|
||||
u64 end, bool skip_sync)
|
||||
{
|
||||
int i;
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
|
||||
av8l_fast_iopte *pmdp = data->pmds;
|
||||
av8l_fast_iopte *pmdp = iopte_pmd_offset(data->pmds, data->base, base);
|
||||
|
||||
for (i = 0; i < ((SZ_1G * 4UL) >> AV8L_FAST_PAGE_SHIFT); ++i) {
|
||||
for (i = base >> AV8L_FAST_PAGE_SHIFT;
|
||||
i <= (end >> AV8L_FAST_PAGE_SHIFT); ++i) {
|
||||
if (!(*pmdp & AV8L_FAST_PTE_VALID)) {
|
||||
*pmdp = 0;
|
||||
if (!skip_sync)
|
||||
|
|
@ -247,7 +250,7 @@ static int av8l_fast_map(struct io_pgtable_ops *ops, unsigned long iova,
|
|||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
struct io_pgtable *iop = iof_pgtable_ops_to_pgtable(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
unsigned long i, nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
av8l_fast_iopte pte;
|
||||
|
||||
|
|
@ -280,7 +283,7 @@ __av8l_fast_unmap(struct io_pgtable_ops *ops, unsigned long iova,
|
|||
? AV8L_FAST_PTE_UNMAPPED_NEED_TLBI
|
||||
: 0;
|
||||
|
||||
ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
nptes = size >> AV8L_FAST_PAGE_SHIFT;
|
||||
|
||||
memset(ptep, val, sizeof(*ptep) * nptes);
|
||||
|
|
@ -378,7 +381,7 @@ static bool av8l_fast_iova_coherent(struct io_pgtable_ops *ops,
|
|||
unsigned long iova)
|
||||
{
|
||||
struct av8l_fast_io_pgtable *data = iof_pgtable_ops_to_data(ops);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, iova);
|
||||
av8l_fast_iopte *ptep = iopte_pmd_offset(data->pmds, data->base, iova);
|
||||
|
||||
return ((PTE_MAIR_IDX(*ptep) == AV8L_FAST_MAIR_ATTR_IDX_CACHE) &&
|
||||
((PTE_SH_IDX(*ptep) == AV8L_FAST_PTE_SH_OS) ||
|
||||
|
|
@ -413,7 +416,7 @@ av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
|
|||
}
|
||||
|
||||
/*
|
||||
* We need 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
|
||||
* We need max 1 page for the pgd, 4 pages for puds (1GB VA per pud page) and
|
||||
* 2048 pages for pmds (each pud page contains 512 table entries, each
|
||||
* pointing to a pmd).
|
||||
*/
|
||||
|
|
@ -422,12 +425,39 @@ av8l_fast_alloc_pgtable_data(struct io_pgtable_cfg *cfg)
|
|||
#define NUM_PMD_PAGES 2048
|
||||
#define NUM_PGTBL_PAGES (NUM_PGD_PAGES + NUM_PUD_PAGES + NUM_PMD_PAGES)
|
||||
|
||||
/* undefine arch specific definitions which depends on page table format */
|
||||
#undef pud_index
|
||||
#undef pud_mask
|
||||
#undef pud_next
|
||||
#undef pmd_index
|
||||
#undef pmd_mask
|
||||
#undef pmd_next
|
||||
|
||||
#define pud_index(addr) (((addr) >> 30) & 0x3)
|
||||
#define pud_mask(addr) ((addr) & ~((1UL << 30) - 1))
|
||||
#define pud_next(addr, end) \
|
||||
({ unsigned long __boundary = pud_mask(addr + (1UL << 30));\
|
||||
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
||||
})
|
||||
|
||||
#define pmd_index(addr) (((addr) >> 21) & 0x1ff)
|
||||
#define pmd_mask(addr) ((addr) & ~((1UL << 21) - 1))
|
||||
#define pmd_next(addr, end) \
|
||||
({ unsigned long __boundary = pmd_mask(addr + (1UL << 21));\
|
||||
(__boundary - 1 < (end) - 1) ? __boundary : (end); \
|
||||
})
|
||||
|
||||
static int
|
||||
av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
||||
struct io_pgtable_cfg *cfg, void *cookie)
|
||||
{
|
||||
int i, j, pg = 0;
|
||||
struct page **pages, *page;
|
||||
dma_addr_t pud, pmd;
|
||||
int pmd_pg_index;
|
||||
struct msm_io_pgtable_info *pgtbl_info = to_msm_io_pgtable_info(cfg);
|
||||
dma_addr_t base = pgtbl_info->iova_base;
|
||||
dma_addr_t end = pgtbl_info->iova_end;
|
||||
|
||||
pages = kmalloc(sizeof(*pages) * NUM_PGTBL_PAGES, __GFP_NOWARN |
|
||||
__GFP_NORETRY);
|
||||
|
|
@ -445,10 +475,11 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
|||
data->pgd = page_address(page);
|
||||
|
||||
/*
|
||||
* We need 2048 entries at level 2 to map 4GB of VA space. A page
|
||||
* can hold 512 entries, so we need 4 pages.
|
||||
* We need max 2048 entries at level 2 to map 4GB of VA space. A page
|
||||
* can hold 512 entries, so we need max 4 pages.
|
||||
*/
|
||||
for (i = 0; i < 4; ++i) {
|
||||
for (i = pud_index(base), pud = base; pud < end;
|
||||
++i, pud = pud_next(pud, end)) {
|
||||
av8l_fast_iopte pte, *ptep;
|
||||
|
||||
page = alloc_page(GFP_KERNEL | __GFP_ZERO);
|
||||
|
|
@ -463,12 +494,15 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
|||
av8l_clean_range(cfg, data->pgd, data->pgd + 4);
|
||||
|
||||
/*
|
||||
* We have 4 puds, each of which can point to 512 pmds, so we'll
|
||||
* have 2048 pmds, each of which can hold 512 ptes, for a grand
|
||||
* We have max 4 puds, each of which can point to 512 pmds, so we'll
|
||||
* have max 2048 pmds, each of which can hold 512 ptes, for a grand
|
||||
* total of 2048*512=1048576 PTEs.
|
||||
*/
|
||||
for (i = 0; i < 4; ++i) {
|
||||
for (j = 0; j < 512; ++j) {
|
||||
pmd_pg_index = pg;
|
||||
for (i = pud_index(base), pud = base; pud < end;
|
||||
++i, pud = pud_next(pud, end)) {
|
||||
for (j = pmd_index(pud), pmd = pud; pmd < pud_next(pud, end);
|
||||
++j, pmd = pmd_next(pmd, end)) {
|
||||
av8l_fast_iopte pte, *pudp;
|
||||
void *addr;
|
||||
|
||||
|
|
@ -487,21 +521,21 @@ av8l_fast_prepopulate_pgtables(struct av8l_fast_io_pgtable *data,
|
|||
av8l_clean_range(cfg, data->puds[i], data->puds[i] + 512);
|
||||
}
|
||||
|
||||
if (WARN_ON(pg != NUM_PGTBL_PAGES))
|
||||
goto err_free_pages;
|
||||
|
||||
/*
|
||||
* We map the pmds into a virtually contiguous space so that we
|
||||
* don't have to traverse the first two levels of the page tables
|
||||
* to find the appropriate pud. Instead, it will be a simple
|
||||
* offset from the virtual base of the pmds.
|
||||
*/
|
||||
data->pmds = vmap(&pages[NUM_PGD_PAGES + NUM_PUD_PAGES], NUM_PMD_PAGES,
|
||||
data->pmds = vmap(&pages[pmd_pg_index], pg - pmd_pg_index,
|
||||
VM_IOREMAP, PAGE_KERNEL);
|
||||
if (!data->pmds)
|
||||
goto err_free_pages;
|
||||
|
||||
data->pages = pages;
|
||||
data->base = base;
|
||||
data->end = end;
|
||||
data->nr_pages = pg;
|
||||
return 0;
|
||||
|
||||
err_free_pages:
|
||||
|
|
@ -605,7 +639,7 @@ static void av8l_fast_free_pgtable(struct io_pgtable *iop)
|
|||
struct av8l_fast_io_pgtable *data = iof_pgtable_to_data(iop);
|
||||
|
||||
vunmap(data->pmds);
|
||||
for (i = 0; i < NUM_PGTBL_PAGES; ++i)
|
||||
for (i = 0; i < data->nr_pages; ++i)
|
||||
__free_page(data->pages[i]);
|
||||
kvfree(data->pages);
|
||||
kfree(data);
|
||||
|
|
@ -674,12 +708,15 @@ static int __init av8l_fast_positive_testing(void)
|
|||
int failed = 0;
|
||||
u64 iova;
|
||||
struct io_pgtable_ops *ops;
|
||||
struct io_pgtable_cfg cfg;
|
||||
struct msm_io_pgtable_info pgtable_info;
|
||||
struct av8l_fast_io_pgtable *data;
|
||||
av8l_fast_iopte *pmds;
|
||||
u64 max = SZ_1G * 4ULL - 1;
|
||||
u64 base = 0;
|
||||
|
||||
cfg = (struct io_pgtable_cfg) {
|
||||
pgtable_info.iova_base = base;
|
||||
pgtable_info.iova_end = max;
|
||||
pgtable_info.pgtbl_cfg = (struct io_pgtable_cfg) {
|
||||
.quirks = 0,
|
||||
.tlb = &dummy_tlb_ops,
|
||||
.ias = 32,
|
||||
|
|
@ -688,8 +725,9 @@ static int __init av8l_fast_positive_testing(void)
|
|||
.coherent_walk = true,
|
||||
};
|
||||
|
||||
cfg_cookie = &cfg;
|
||||
ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &cfg, &cfg);
|
||||
cfg_cookie = &pgtable_info.pgtbl_cfg;
|
||||
ops = alloc_io_pgtable_ops(ARM_V8L_FAST, &pgtable_info.pgtbl_cfg,
|
||||
&pgtable_info.pgtbl_cfg);
|
||||
|
||||
if (WARN_ON(!ops))
|
||||
return 1;
|
||||
|
|
@ -698,81 +736,81 @@ static int __init av8l_fast_positive_testing(void)
|
|||
pmds = data->pmds;
|
||||
|
||||
/* map the entire 4GB VA space with 4K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_4K) {
|
||||
for (iova = base; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_4K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = 0; iova < max; iova += SZ_4K) {
|
||||
for (iova = base; iova < max; iova += SZ_4K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_4K, NULL) != SZ_4K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
av8l_fast_clear_stale_ptes(ops, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 8K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_8K) {
|
||||
for (iova = base; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_8K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all with 8K unmap calls */
|
||||
for (iova = 0; iova < max; iova += SZ_8K) {
|
||||
for (iova = base; iova < max; iova += SZ_8K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_8K, NULL) != SZ_8K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
av8l_fast_clear_stale_ptes(ops, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 16K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_16K) {
|
||||
for (iova = base; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_16K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all */
|
||||
for (iova = 0; iova < max; iova += SZ_16K) {
|
||||
for (iova = base; iova < max; iova += SZ_16K) {
|
||||
if (WARN_ON(ops->unmap(ops, iova, SZ_16K, NULL) != SZ_16K))
|
||||
failed++;
|
||||
}
|
||||
|
||||
/* sweep up TLB proving PTEs */
|
||||
av8l_fast_clear_stale_ptes(ops, false);
|
||||
av8l_fast_clear_stale_ptes(ops, base, max, false);
|
||||
|
||||
/* map the entire 4GB VA space with 64K map calls */
|
||||
for (iova = 0; iova < max; iova += SZ_64K) {
|
||||
for (iova = base; iova < max; iova += SZ_64K) {
|
||||
if (WARN_ON(ops->map(ops, iova, iova, SZ_64K, IOMMU_READ))) {
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, 0, 0,
|
||||
max)))
|
||||
if (WARN_ON(!av8l_fast_range_has_specific_mapping(ops, base,
|
||||
base, max - base)))
|
||||
failed++;
|
||||
|
||||
/* unmap it all at once */
|
||||
if (WARN_ON(ops->unmap(ops, 0, max, NULL) != max))
|
||||
if (WARN_ON(ops->unmap(ops, base, max - base, NULL) != (max - base)))
|
||||
failed++;
|
||||
|
||||
free_io_pgtable_ops(ops);
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ struct av8l_fast_io_pgtable {
|
|||
av8l_fast_iopte *puds[4];
|
||||
av8l_fast_iopte *pmds;
|
||||
struct page **pages; /* page table memory */
|
||||
int nr_pages;
|
||||
dma_addr_t base;
|
||||
dma_addr_t end;
|
||||
};
|
||||
|
||||
/* Struct accessors */
|
||||
|
|
@ -99,7 +102,8 @@ av8l_fast_iova_to_phys_public(struct io_pgtable_ops *ops,
|
|||
*/
|
||||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0xa
|
||||
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, bool skip_sync);
|
||||
void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops, u64 base, u64 end,
|
||||
bool skip_sync);
|
||||
void av8l_register_notify(struct notifier_block *nb);
|
||||
|
||||
#else /* !CONFIG_IOMMU_IO_PGTABLE_FAST_PROVE_TLB */
|
||||
|
|
@ -107,6 +111,8 @@ void av8l_register_notify(struct notifier_block *nb);
|
|||
#define AV8L_FAST_PTE_UNMAPPED_NEED_TLBI 0
|
||||
|
||||
static inline void av8l_fast_clear_stale_ptes(struct io_pgtable_ops *ops,
|
||||
u64 base,
|
||||
u64 end,
|
||||
bool skip_sync)
|
||||
{
|
||||
}
|
||||
|
|
|
|||
|
|
@ -190,6 +190,8 @@ struct io_pgtable_ops {
|
|||
* and False if non-coherent.
|
||||
* @iova_to_pte: Translate iova to Page Table Entry (PTE).
|
||||
* @pgtbl_cfg: The configuration for a set of page tables.
|
||||
* @iova_base: Configured IOVA base
|
||||
* @iova_end: Configured IOVA end
|
||||
*/
|
||||
struct msm_io_pgtable_info {
|
||||
int (*map_sg)(struct io_pgtable_ops *ops, unsigned long iova,
|
||||
|
|
@ -199,6 +201,8 @@ struct msm_io_pgtable_info {
|
|||
unsigned long iova);
|
||||
uint64_t (*iova_to_pte)(struct io_pgtable_ops *ops, unsigned long iova);
|
||||
struct io_pgtable_cfg pgtbl_cfg;
|
||||
dma_addr_t iova_base;
|
||||
dma_addr_t iova_end;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
Loading…
Reference in a new issue