Keeping PTE-level mappings is interesting for arm64 as it allows mapping/unmapping pages from the kernel direct map without the risk of splitting blocks which, under the break-before-make rule, may trigger page-faults the kernel can't handle.

However, mapping the entire direct map at PTE-level is costly. So instead, create a new memblock flag MEMBLOCK_PTEMAP to selectively apply a PTE-level mapping.

Signed-off-by: Vincent Donnefort vdonnefort@google.com --- include/linux/memblock.h | 9 +++++++++ mm/memblock.c | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 40e6e0aa2f8d..67ef20a1c5e2 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -52,6 +52,7 @@ extern unsigned long long max_possible_pfn; * know is good to use. It is the only memory that allocations may happen from * in this phase. * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages + * @MEMBLOCK_PTEMAP: memory region to be mapped using PTE-level mapping */ enum memblock_flags { MEMBLOCK_NONE = 0x0, /* No special request */ @@ -63,6 +64,7 @@ enum memblock_flags { MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */ MEMBLOCK_KHO_NOPRSRV = 0x40, /* memory with no KHO preservations */ MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */ + MEMBLOCK_PTEMAP = 0x100,/* PTE-level mapping */ };

/** @@ -160,6 +162,8 @@ int memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size); int memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size); int memblock_mark_kho_noprsrv(phys_addr_t base, phys_addr_t size); int memblock_clear_kho_noprsrv(phys_addr_t base, phys_addr_t size); +int memblock_mark_ptemap(phys_addr_t base, phys_addr_t size); +int memblock_clear_ptemap(phys_addr_t base, phys_addr_t size);

void memblock_free(void *ptr, size_t size);

@@ -305,6 +309,11 @@ static inline bool memblock_is_kho_noprsrv(struct memblock_region *m) return m->flags & MEMBLOCK_KHO_NOPRSRV; }

+static inline bool memblock_is_ptemap(struct memblock_region *m) +{ + return m->flags & MEMBLOCK_PTEMAP; +} + int memblock_search_pfn_nid(unsigned long pfn, unsigned long *start_pfn, unsigned long *end_pfn); void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, diff --git a/mm/memblock.c b/mm/memblock.c index a450092a80fb..b8b6e762917a 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1068,10 +1068,15 @@ static int __init_memblock memblock_setclr_flag(struct memblock_type *type, for (i = start_rgn; i < end_rgn; i++) { struct memblock_region *r = &type->regions[i];

- if (set) + if (set) { + enum memblock_flags map_flags = + MEMBLOCK_NOMAP | MEMBLOCK_PTEMAP; + r->flags |= flag; - else + WARN_ON_ONCE((r->flags & map_flags) == map_flags); + } else { r->flags &= ~flag; + } }

memblock_merge_regions(type, start_rgn, end_rgn); @@ -1221,6 +1226,35 @@ __init int memblock_clear_kho_noprsrv(phys_addr_t base, phys_addr_t size) MEMBLOCK_KHO_NOPRSRV); }

+/** + * memblock_mark_ptemap - Mark a memory region with flag MEMBLOCK_PTEMAP. + * @base: the base phys addr of the region + * @size: the size of the region + * + * If supported by the architecture, such region is mapped using PTE-level + * mappings in the kernel direct map. + * + * Return: 0 on success, -errno on failure. + */ +int __init_memblock memblock_mark_ptemap(phys_addr_t base, phys_addr_t size) +{ + return memblock_setclr_flag(&memblock.memory, base, size, 1, + MEMBLOCK_PTEMAP); +} + +/** + * memblock_clear_ptemap - Clear flag MEMBLOCK_PTEMAP for a specified region. + * @base: the base phys addr of the region + * @size: the size of the region + * + * Return: 0 on success, -errno on failure. + */ +int __init_memblock memblock_clear_ptemap(phys_addr_t base, phys_addr_t size) +{ + return memblock_setclr_flag(&memblock.memory, base, size, 0, + MEMBLOCK_PTEMAP); +} + static bool should_skip_region(struct memblock_type *type, struct memblock_region *m, int nid, int flags) @@ -2854,6 +2888,7 @@ static const char * const flagname[] = { [ilog2(MEMBLOCK_RSRV_KERN)] = "RSV_KERN", [ilog2(MEMBLOCK_KHO_NOPRSRV)] = "KHO_NOPRSRV", [ilog2(MEMBLOCK_RSRV_HUGETLB)] = "RSV_HUGETLB", + [ilog2(MEMBLOCK_PTEMAP)] = "PTEMAP", };

static int memblock_debug_show(struct seq_file *m, void *private)

Expose rmem_cma_ops via a kernel/dma-private header so the upcoming lendable-dma-pool reserved-memory handler can wrap the CMA callbacks instead of duplicating them.

Signed-off-by: Vincent Donnefort vdonnefort@google.com --- kernel/dma/contiguous.c | 4 +++- kernel/dma/contiguous.h | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 kernel/dma/contiguous.h

diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c index 90937dc7c627..22d6e54aea35 100644 --- a/kernel/dma/contiguous.c +++ b/kernel/dma/contiguous.c @@ -46,6 +46,8 @@ #include <linux/cma.h> #include <linux/nospec.h>

+#include "contiguous.h" + #ifdef CONFIG_CMA_SIZE_MBYTES #define CMA_SIZE_MBYTES CONFIG_CMA_SIZE_MBYTES #else @@ -578,7 +580,7 @@ static int __init rmem_cma_setup(unsigned long node, struct reserved_mem *rmem) return 0; }

-static const struct reserved_mem_ops rmem_cma_ops = { +const struct reserved_mem_ops rmem_cma_ops = { .node_validate = rmem_cma_validate, .node_fixup = rmem_cma_fixup, .node_init = rmem_cma_setup, diff --git a/kernel/dma/contiguous.h b/kernel/dma/contiguous.h new file mode 100644 index 000000000000..9b1fdbfc794f --- /dev/null +++ b/kernel/dma/contiguous.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KERNEL_DMA_CONTIGUOUS_H +#define _KERNEL_DMA_CONTIGUOUS_H + +#ifdef CONFIG_OF_RESERVED_MEM +struct reserved_mem_ops; +extern const struct reserved_mem_ops rmem_cma_ops; +#endif + +#endif /* _KERNEL_DMA_CONTIGUOUS_H */

On arm64, when memory is lent to secure world (e.g. via Arm FF-A), fatal CPU speculative reads can still occur while it retains a cacheable direct mapping. Introduce the "lendable-dma-pool" reserved-memory CMA driver with dma_zap_direct_map() and dma_restore_direct_map() to unmap pages before lending and restore them upon reclaim. Devices use the pool via "memory-region" or "linux,cma-default":

reserved-memory { #address-cells = <0x2>; #size-cells = <0x2>; ranges;

linux,cma { compatible = "lendable-dma-pool"; reusable; linux,cma-default; size = <0x0 0x4000000>; }; };

If no lendable-dma-pool node is provided, the API falls back to can_set_direct_map().

Signed-off-by: Vincent Donnefort vdonnefort@google.com --- include/linux/dma-map-ops.h | 9 ++ include/linux/dma-mapping.h | 16 +++ kernel/dma/Kconfig | 3 + kernel/dma/Makefile | 1 + kernel/dma/lendable.c | 243 ++++++++++++++++++++++++++++++++++++ 5 files changed, 272 insertions(+) create mode 100644 kernel/dma/lendable.c

diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index afaf2c12189b..e1d1e8d70855 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -379,6 +379,15 @@ static inline void arch_dma_prep_coherent(struct page *page, size_t size) } #endif /* CONFIG_ARCH_HAS_DMA_PREP_COHERENT */

+#ifdef CONFIG_DMA_LENDABLE +int __init arch_dma_mark_lendable(phys_addr_t base, phys_addr_t size); +bool __init arch_dma_is_lendable(struct page *page, unsigned int nr_pages); +int arch_dma_zap_direct_map(struct page *page, unsigned int nr_pages, + bool from_lendable); +int arch_dma_restore_direct_map(struct page *page, unsigned int nr_pages, + bool from_lendable); +#endif /* CONFIG_DMA_LENDABLE */ + void *arch_dma_set_uncached(void *addr, size_t size); void arch_dma_clear_uncached(void *addr, size_t size);

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index efde0e5a58dc..24239599505d 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -511,6 +511,22 @@ void dma_free_pages(struct device *dev, size_t size, struct page *page, int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, size_t size, struct page *page);

+#ifdef CONFIG_DMA_LENDABLE +int dma_zap_direct_map(struct device *dev, struct page *page, size_t size); +void dma_restore_direct_map(struct device *dev, struct page *page, size_t size); +#else +static inline int dma_zap_direct_map(struct device *dev, struct page *page, + size_t size) +{ + return -ENODEV; +} + +static inline void dma_restore_direct_map(struct device *dev, struct page *page, + size_t size) +{ +} +#endif /* CONFIG_DMA_LENDABLE */ + static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) { diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig index 3830a63ae032..8d65e2daa7d2 100644 --- a/kernel/dma/Kconfig +++ b/kernel/dma/Kconfig @@ -192,6 +192,9 @@ config DMA_CMA

if DMA_CMA

+config DMA_LENDABLE + bool + config DMA_NUMA_CMA bool "Enable separate DMA Contiguous Memory Area for NUMA Node" depends on NUMA diff --git a/kernel/dma/Makefile b/kernel/dma/Makefile index 6977033444a3..6b257d82bd9b 100644 --- a/kernel/dma/Makefile +++ b/kernel/dma/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_HAS_DMA) += mapping.o direct.o obj-$(CONFIG_DMA_OPS_HELPERS) += ops_helpers.o obj-$(CONFIG_ARCH_HAS_DMA_OPS) += dummy.o obj-$(CONFIG_DMA_CMA) += contiguous.o +obj-$(CONFIG_DMA_LENDABLE) += lendable.o obj-$(CONFIG_DMA_DECLARE_COHERENT) += coherent.o obj-$(CONFIG_DMA_API_DEBUG) += debug.o obj-$(CONFIG_SWIOTLB) += swiotlb.o diff --git a/kernel/dma/lendable.c b/kernel/dma/lendable.c new file mode 100644 index 000000000000..85ac3ad10df0 --- /dev/null +++ b/kernel/dma/lendable.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Lendable DMA Contiguous Memory Pool + * + * Prevents CPU speculative reads to DMA memory (e.g. when lent to a secure + * world) by unmapping it from the kernel direct map. This works if the + * architecture allows modifying the direct map globally (can_set_direct_map()) + * or if the memory belongs to a reserved-memory CMA pool compatible with + * "lendable-dma-pool". + * + * Copyright (C) 2026 Google LLC + * Author: Vincent Donnefort vdonnefort@google.com + */ + +#define pr_fmt(fmt) "dma-lendable: " fmt + +#include <linux/atomic.h> +#include <linux/bsearch.h> +#include <linux/cma.h> +#include <linux/dma-map-ops.h> +#include <linux/dma-mapping.h> +#include <linux/init.h> +#include <linux/mm.h> +#include <linux/of_reserved_mem.h> +#include <linux/set_memory.h> +#include <linux/sort.h> +#include <linux/suspend.h> + +#include "contiguous.h" + +#define DMA_LENDABLE_DISABLED -1 + +static struct cma *lendable_areas[MAX_CMA_AREAS] __ro_after_init; +static unsigned int nr_lendable_areas __ro_after_init; +static atomic_t dma_lendable_active; + +static int __lendable_cmp(const void *a, const void *b) +{ + const struct cma *cma_a = *(struct cma * const *)a; + const struct cma *cma_b = *(struct cma * const *)b; + + return cmp_int(cma_get_base(cma_a), cma_get_base(cma_b)); +} + +static bool cma_is_lendable(struct cma *cma) +{ + return cma && bsearch(&cma, lendable_areas, nr_lendable_areas, + sizeof(lendable_areas[0]), __lendable_cmp); +} + +static bool cma_has_range(struct cma *cma, struct page *page, unsigned int nr_pages) +{ + phys_addr_t start = page_to_phys(page); + phys_addr_t end = start + ((phys_addr_t)nr_pages << PAGE_SHIFT); + phys_addr_t base = cma_get_base(cma); + + return start >= base && end <= base + cma_get_size(cma); +} + +/** + * dma_zap_direct_map() - Remove a DMA page range from the kernel direct map + * @dev: Device that allocated the pages (or NULL for default CMA pool) + * @page: First page of the memory region + * @size: Page-aligned size of the memory region in bytes + * + * Cleans and invalidates caches and removes the memory range from the kernel + * direct map to prevent speculative CPU accesses. Must be paired with + * dma_restore_direct_map(). + * + * Context: Any context if allocated from a lendable pool; process context + * (may sleep) otherwise. + * Return: + * * 0 - Success. + * * -EINVAL - @size is 0 or not page-aligned. + * * -ENODEV - Direct map cannot be modified and @dev has no lendable pool. + * * -EFAULT - @dev has a lendable pool, but [@page, @page + @size) is + * outside that pool. + * * -EBUSY - Direct map zapping is temporarily disabled (hibernation). + */ +int dma_zap_direct_map(struct device *dev, struct page *page, size_t size) +{ + unsigned int nr_pages = size >> PAGE_SHIFT; + bool from_lendable, has_lendable; + struct cma *cma; + int ret; + + if (!size || !PAGE_ALIGNED(size)) + return -EINVAL; + + cma = dev_get_cma_area(dev); + has_lendable = cma_is_lendable(cma); + from_lendable = has_lendable && cma_has_range(cma, page, nr_pages); + + if (!from_lendable && !can_set_direct_map()) + return has_lendable ? -EFAULT : -ENODEV; + + /* Provides full ordering */ + if (!atomic_add_unless(&dma_lendable_active, 1, DMA_LENDABLE_DISABLED)) + return -EBUSY; + + ret = arch_dma_zap_direct_map(page, nr_pages, from_lendable); + if (ret) + atomic_dec(&dma_lendable_active); + + return ret; +} +EXPORT_SYMBOL_GPL(dma_zap_direct_map); + +/** + * dma_restore_direct_map() - Restore a zapped DMA page range in the direct map + * @dev: Device that allocated the pages (or NULL for default CMA pool) + * @page: First page of the memory region + * @size: Page-aligned size of the memory region in bytes + * + * Restores the default kernel direct mapping and invalidates caches for a range + * previously zapped with dma_zap_direct_map(). + * + * Context: Any context if allocated from a lendable pool; process context + * (may sleep) otherwise. + */ +void dma_restore_direct_map(struct device *dev, struct page *page, size_t size) +{ + unsigned int nr_pages = size >> PAGE_SHIFT; + bool from_lendable; + struct cma *cma; + + if (WARN_ON_ONCE(!size || !PAGE_ALIGNED(size))) + return; + + cma = dev_get_cma_area(dev); + from_lendable = cma_is_lendable(cma) && cma_has_range(cma, page, nr_pages); + + if (WARN_ON_ONCE(!from_lendable && !can_set_direct_map())) + return; + + if (WARN_ON_ONCE(arch_dma_restore_direct_map(page, nr_pages, from_lendable))) + return; + + WARN_ON_ONCE(atomic_dec_return_release(&dma_lendable_active) < 0); +} +EXPORT_SYMBOL_GPL(dma_restore_direct_map); + +static int dma_lendable_pm_notify(struct notifier_block *nb, + unsigned long mode, void *data) +{ + /* Prevent hibernation while unmapped pages may be inaccessible */ + switch (mode) { + case PM_HIBERNATION_PREPARE: + case PM_RESTORE_PREPARE: + if (atomic_cmpxchg_acquire(&dma_lendable_active, 0, DMA_LENDABLE_DISABLED)) + return notifier_from_errno(-EBUSY); + break; + case PM_POST_HIBERNATION: + case PM_POST_RESTORE: + atomic_cmpxchg(&dma_lendable_active, DMA_LENDABLE_DISABLED, 0); + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block dma_lendable_pm_nb = { + .notifier_call = dma_lendable_pm_notify, +}; + +static int __init dma_lendable_init(void) +{ + unsigned int i, nr = 0; + + for (i = 0; i < nr_lendable_areas; i++) { + struct cma *cma = lendable_areas[i]; + + if (!arch_dma_is_lendable(phys_to_page(cma_get_base(cma)), + cma_get_size(cma) >> PAGE_SHIFT)) { + pr_warn("Ignoring %s: not lendable\n", cma_get_name(cma)); + continue; + } + + lendable_areas[nr++] = cma; + } + nr_lendable_areas = nr; + + sort(lendable_areas, nr_lendable_areas, sizeof(lendable_areas[0]), __lendable_cmp, NULL); + + register_pm_notifier(&dma_lendable_pm_nb); + + return 0; +} +core_initcall(dma_lendable_init); + +#ifdef CONFIG_OF_RESERVED_MEM +static int __init rmem_lendable_validate(unsigned long node, + phys_addr_t *align) +{ + return rmem_cma_ops.node_validate(node, align); +} + +static int __init rmem_lendable_fixup(unsigned long node, phys_addr_t base, + phys_addr_t size) +{ + int ret = rmem_cma_ops.node_fixup(node, base, size); + + if (ret) + return ret; + + return arch_dma_mark_lendable(base, size); +} + +static int __init rmem_lendable_setup(unsigned long node, + struct reserved_mem *rmem) +{ + int ret = rmem_cma_ops.node_init(node, rmem); + + if (ret) + return ret; + + lendable_areas[nr_lendable_areas++] = rmem->priv; + + return 0; +} + +static int rmem_lendable_device_init(struct reserved_mem *rmem, + struct device *dev) +{ + return rmem_cma_ops.device_init(rmem, dev); +} + +static void rmem_lendable_device_release(struct reserved_mem *rmem, + struct device *dev) +{ + rmem_cma_ops.device_release(rmem, dev); +} + +static const struct reserved_mem_ops rmem_lendable_ops = { + .node_validate = rmem_lendable_validate, + .node_fixup = rmem_lendable_fixup, + .node_init = rmem_lendable_setup, + .device_init = rmem_lendable_device_init, + .device_release = rmem_lendable_device_release, +}; + +RESERVEDMEM_OF_DECLARE(cma_lendable, "lendable-dma-pool", &rmem_lendable_ops); +#endif /* CONFIG_OF_RESERVED_MEM */

Regions marked with MEMBLOCK_PTEMAP are mapped at page granularity so their direct mapping can be modified without splitting blocks at runtime. Introduce direct_map_is_ptemap(), which walks the kernel page tables during init to verify that every page in a range is backed by a present non-contiguous PTE.

Signed-off-by: Vincent Donnefort vdonnefort@google.com --- arch/arm64/include/asm/set_memory.h | 4 ++ arch/arm64/mm/pageattr.c | 65 +++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+)

diff --git a/arch/arm64/include/asm/set_memory.h b/arch/arm64/include/asm/set_memory.h index 0091ba12200e..b6ec938703d6 100644 --- a/arch/arm64/include/asm/set_memory.h +++ b/arch/arm64/include/asm/set_memory.h @@ -3,12 +3,16 @@ #ifndef _ASM_ARM64_SET_MEMORY_H #define _ASM_ARM64_SET_MEMORY_H

+#include <linux/init.h> + #include <asm/mem_encrypt.h> #include <asm-generic/set_memory.h>

bool can_set_direct_map(void); #define can_set_direct_map can_set_direct_map

+bool __init direct_map_is_ptemap(struct page *page, unsigned int numpages); + int set_memory_valid(unsigned long addr, int numpages, int enable);

int set_direct_map_invalid_noflush(struct page *page, unsigned int numpages); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index de7bf95003ea..c7662fd2fcb2 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -102,6 +102,71 @@ bool can_set_direct_map(void) arm64_kfence_can_set_direct_map() || is_realm_world(); }

+static int __init direct_map_is_ptemap_pud_entry(pud_t *pud, unsigned long addr, + unsigned long next, + struct mm_walk *walk) +{ + if (pud_leaf(pudp_get(pud))) + return -EINVAL; + + return 0; +} + +static int __init direct_map_is_ptemap_pmd_entry(pmd_t *pmd, unsigned long addr, + unsigned long next, + struct mm_walk *walk) +{ + if (pmd_leaf(pmdp_get(pmd))) + return -EINVAL; + + return 0; +} + +static int __init direct_map_is_ptemap_pte_entry(pte_t *pte, unsigned long addr, + unsigned long next, + struct mm_walk *walk) +{ + pte_t val = __ptep_get(pte); + + if (!pte_present(val) || pte_cont(val)) + return -EINVAL; + + return 0; +} + +static int __init direct_map_is_ptemap_pte_hole(unsigned long addr, + unsigned long next, int depth, + struct mm_walk *walk) +{ + return -EINVAL; +} + +static const struct mm_walk_ops direct_map_is_ptemap_ops __initconst = { + .pud_entry = direct_map_is_ptemap_pud_entry, + .pmd_entry = direct_map_is_ptemap_pmd_entry, + .pte_entry = direct_map_is_ptemap_pte_entry, + .pte_hole = direct_map_is_ptemap_pte_hole, +}; + +bool __init direct_map_is_ptemap(struct page *page, unsigned int numpages) +{ + unsigned long addr, end; + + if (!pfn_valid(page_to_pfn(page)) || !numpages) + return false; + + addr = (unsigned long)page_address(page); + end = addr + PAGE_SIZE * numpages; + + /* + * Called during boot before runtime linear-map splitting, so it is + * safe to walk the page-table lockless. + */ + return !walk_kernel_page_table_range_lockless(addr, end, + &direct_map_is_ptemap_ops, + NULL, NULL); +} + static int update_range_prot(unsigned long start, unsigned long size, pgprot_t set_mask, pgprot_t clear_mask) {

When memory is lent to secure world, fatal CPU speculative reads can still occur while it retains a cacheable mapping. Use the lendable-dma-pool API to unmap pages before lending and restore them upon reclaim.

Signed-off-by: Vincent Donnefort vdonnefort@google.com --- drivers/tee/optee/protmem.c | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-)

diff --git a/drivers/tee/optee/protmem.c b/drivers/tee/optee/protmem.c index be3abf6e8aa6..10e8f1540e5e 100644 --- a/drivers/tee/optee/protmem.c +++ b/drivers/tee/optee/protmem.c @@ -4,6 +4,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

+#include <linux/dma-mapping.h> #include <linux/errno.h> #include <linux/genalloc.h> #include <linux/slab.h> @@ -21,6 +22,7 @@ struct optee_protmem_dyn_pool { u_int mem_attr_count; refcount_t refcount; u32 use_case; + bool unmapped; struct tee_shm *protmem; /* Protects when initializing and tearing down this struct */ struct mutex mutex; @@ -32,6 +34,35 @@ to_protmem_dyn_pool(struct tee_protmem_pool *pool) return container_of(pool, struct optee_protmem_dyn_pool, pool); }

+static int zap_direct_map_dyn_protmem(struct optee_protmem_dyn_pool *rp) +{ + int ret; + + ret = dma_zap_direct_map(&rp->optee->teedev->dev, + phys_to_page(rp->protmem->paddr), + rp->protmem->size); + if (ret && ret != -ENODEV) + return ret; + + rp->unmapped = !ret; + + return 0; +} + +static void restore_direct_map_dyn_protmem(struct optee_protmem_dyn_pool *rp) +{ + if (WARN_ON(!rp->protmem)) + return; + + if (!rp->unmapped) + return; + + dma_restore_direct_map(&rp->optee->teedev->dev, + phys_to_page(rp->protmem->paddr), + rp->protmem->size); + rp->unmapped = false; +} + static int init_dyn_protmem(struct optee_protmem_dyn_pool *rp) { int rc; @@ -42,19 +73,15 @@ static int init_dyn_protmem(struct optee_protmem_dyn_pool *rp) goto err_null_protmem; }

- /* - * TODO unmap the memory range since the physical memory will - * become inaccesible after the lend_protmem() call. - * - * If the platform supports a hypervisor at EL2, it will unmap the - * intermediate physical memory for us and stop cache pre-fetch of - * the memory. - */ + rc = zap_direct_map_dyn_protmem(rp); + if (rc) + goto err_put_shm; + rc = rp->optee->ops->lend_protmem(rp->optee, rp->protmem, rp->mem_attrs, rp->mem_attr_count, rp->use_case); if (rc) - goto err_put_shm; + goto err_restore_direct_map; rp->protmem->flags |= TEE_SHM_DYNAMIC;

rp->gen_pool = gen_pool_create(PAGE_SHIFT, -1); @@ -76,6 +103,8 @@ static int init_dyn_protmem(struct optee_protmem_dyn_pool *rp) rp->gen_pool = NULL; err_reclaim: rp->optee->ops->reclaim_protmem(rp->optee, rp->protmem); +err_restore_direct_map: + restore_direct_map_dyn_protmem(rp); err_put_shm: tee_shm_put(rp->protmem); err_null_protmem: @@ -112,6 +141,7 @@ static void release_dyn_protmem(struct optee_protmem_dyn_pool *rp) rp->gen_pool = NULL;

rp->optee->ops->reclaim_protmem(rp->optee, rp->protmem); + restore_direct_map_dyn_protmem(rp); rp->protmem->flags &= ~TEE_SHM_DYNAMIC;

WARN(refcount_read(&rp->protmem->refcount) != 1, "Unexpected refcount");