// SPDX-License-Identifier: GPL-2.0 /* * Implementation of the IOMMU SVA API for the ARM SMMUv3 */ #include #include #include #include #include #include #include "arm-smmu-v3.h" #include "../../io-pgtable-arm.h" static void __maybe_unused arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) { struct arm_smmu_master_domain *master_domain; struct arm_smmu_cd target_cd; unsigned long flags; spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { struct arm_smmu_master *master = master_domain->master; struct arm_smmu_cd *cdptr; cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); if (WARN_ON(!cdptr)) continue; arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, &target_cd); } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); } static u64 page_size_to_cd(void) { static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || PAGE_SIZE == SZ_64K); if (PAGE_SIZE == SZ_64K) return ARM_LPAE_TCR_TG0_64K; if (PAGE_SIZE == SZ_16K) return ARM_LPAE_TCR_TG0_16K; return ARM_LPAE_TCR_TG0_4K; } VISIBLE_IF_KUNIT void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, struct arm_smmu_master *master, struct mm_struct *mm, u16 asid) { u64 par; memset(target, 0, sizeof(*target)); par = cpuid_feature_extract_unsigned_field( read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), ID_AA64MMFR0_EL1_PARANGE_SHIFT); target->data[0] = cpu_to_le64( CTXDESC_CD_0_TCR_EPD1 | #ifdef __BIG_ENDIAN CTXDESC_CD_0_ENDI | #endif CTXDESC_CD_0_V | FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | CTXDESC_CD_0_AA64 | (master->stall_enabled ? CTXDESC_CD_0_S : 0) | CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET | FIELD_PREP(CTXDESC_CD_0_ASID, asid)); /* * If no MM is passed then this creates a SVA entry that faults * everything. arm_smmu_write_cd_entry() can hitlessly go between these * two entries types since TTB0 is ignored by HW when EPD0 is set. */ if (mm) { target->data[0] |= cpu_to_le64( FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & CTXDESC_CD_1_TTB0_MASK); } else { target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); /* * Disable stall and immediately generate an abort if stall * disable is permitted. This speeds up cleanup for an unclean * exit if the device is still doing a lot of DMA. */ if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) target->data[0] &= cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); } /* * MAIR value is pretty much constant and global, so we can just get it * from the current CPU register */ target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); /* * Note that we don't bother with S1PIE on the SMMU, we just rely on * our default encoding scheme matching direct permissions anyway. * SMMU has no notion of S1POE nor GCS, so make sure that is clear if * either is enabled for CPUs, just in case anyone imagines otherwise. */ if (system_supports_poe() || system_supports_gcs()) dev_warn_once(master->smmu->dev, "SVA devices ignore permission overlays and GCS\n"); } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); /* * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this * is used as a threshold to replace per-page TLBI commands to issue in the * command queue with an address-space TLBI command, when SMMU w/o a range * invalidation feature handles too many per-page TLBI commands, which will * otherwise result in a soft lockup. */ #define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long start, unsigned long end) { struct arm_smmu_domain *smmu_domain = container_of(mn, struct arm_smmu_domain, mmu_notifier); size_t size; /* * The mm_types defines vm_end as the first byte after the end address, * different from IOMMU subsystem using the last address of an address * range. So do a simple translation here by calculating size correctly. */ size = end - start; if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) size = 0; } else { if (size == ULONG_MAX) size = 0; } if (!size) arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); else arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, PAGE_SIZE, false, smmu_domain); arm_smmu_atc_inv_domain(smmu_domain, start, size); } static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct arm_smmu_domain *smmu_domain = container_of(mn, struct arm_smmu_domain, mmu_notifier); struct arm_smmu_master_domain *master_domain; unsigned long flags; /* * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, * but disable translation. */ spin_lock_irqsave(&smmu_domain->devices_lock, flags); list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { struct arm_smmu_master *master = master_domain->master; struct arm_smmu_cd target; struct arm_smmu_cd *cdptr; cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); if (WARN_ON(!cdptr)) continue; arm_smmu_make_sva_cd(&target, master, NULL, smmu_domain->cd.asid); arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, &target); } spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); arm_smmu_atc_inv_domain(smmu_domain, 0, 0); } static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) { kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier)); } static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, .release = arm_smmu_mm_release, .free_notifier = arm_smmu_mmu_notifier_free, }; bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) { unsigned long reg, fld; unsigned long oas; unsigned long asid_bits; u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; if (vabits_actual == 52) { /* We don't support LPA2 */ if (PAGE_SIZE != SZ_64K) return false; feat_mask |= ARM_SMMU_FEAT_VAX; } if (system_supports_bbml2_noabort()) feat_mask |= ARM_SMMU_FEAT_BBML2; if ((smmu->features & feat_mask) != feat_mask) return false; if (!(smmu->pgsize_bitmap & PAGE_SIZE)) return false; /* * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're * not even pretending to support AArch32 here. Abort if the MMU outputs * addresses larger than what we support. */ reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); oas = id_aa64mmfr0_parange_to_phys_shift(fld); if (smmu->oas < oas) return false; /* We can support bigger ASIDs than the CPU, but not smaller */ fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); asid_bits = fld ? 16 : 8; if (smmu->asid_bits < asid_bits) return false; /* * See max_pinned_asids in arch/arm64/mm/context.c. The following is * generally the maximum number of bindable processes. */ if (arm64_kernel_unmapped_at_el0()) asid_bits--; dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - num_possible_cpus() - 2); return true; } void arm_smmu_sva_notifier_synchronize(void) { /* * Some MMU notifiers may still be waiting to be freed, using * arm_smmu_mmu_notifier_free(). Wait for them. */ mmu_notifier_synchronize(); } static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, struct device *dev, ioasid_t id, struct iommu_domain *old) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct arm_smmu_cd target; int ret; if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return -EOPNOTSUPP; /* Prevent arm_smmu_mm_release from being called while we are attaching */ if (!mmget_not_zero(domain->mm)) return -EINVAL; /* * This does not need the arm_smmu_asid_lock because SVA domains never * get reassigned */ arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid); ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old); mmput(domain->mm); return ret; } static void arm_smmu_sva_domain_free(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); /* * Ensure the ASID is empty in the iommu cache before allowing reuse. */ arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); /* * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can * still be called/running at this point. We allow the ASID to be * reused, and if there is a race then it just suffers harmless * unnecessary invalidation. */ xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); /* * Actual free is defered to the SRCU callback * arm_smmu_mmu_notifier_free() */ mmu_notifier_put(&smmu_domain->mmu_notifier); } static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { .set_dev_pasid = arm_smmu_sva_set_dev_pasid, .free = arm_smmu_sva_domain_free }; struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, struct mm_struct *mm) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); struct arm_smmu_device *smmu = master->smmu; struct arm_smmu_domain *smmu_domain; u32 asid; int ret; if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) return ERR_PTR(-EOPNOTSUPP); smmu_domain = arm_smmu_domain_alloc(); if (IS_ERR(smmu_domain)) return ERR_CAST(smmu_domain); smmu_domain->domain.type = IOMMU_DOMAIN_SVA; smmu_domain->domain.ops = &arm_smmu_sva_domain_ops; /* * Choose page_size as the leaf page size for invalidation when * ARM_SMMU_FEAT_RANGE_INV is present */ smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; smmu_domain->smmu = smmu; ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); if (ret) goto err_free; smmu_domain->cd.asid = asid; smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops; ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm); if (ret) goto err_asid; return &smmu_domain->domain; err_asid: xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); err_free: kfree(smmu_domain); return ERR_PTR(ret); }